Merge pull request #30 from pingping-lin/ONOS-Next
enable login multiple Quaggas && add new test Case1
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 44a70b8..9c5ad4d 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -42,7 +42,7 @@
logmsg = logmsg + "\n\tTest Script :" + path + "Tests/" + main.TEST + ".py"+ ""
logmsg = logmsg + "\n\tTest Params : " + path + "Tests/" + main.TEST + ".params" + ""
- logmsg = logmsg + "\n\tTopology : " + path + "Tests/" +main.TEST + ".tpl" + ""
+ logmsg = logmsg + "\n\tTopology : " + path + "Tests/" +main.TEST + ".topo" + ""
logmsg = logmsg + "\n"+" " * 30+"+" +"-" * 18+"+" +"\n" +"-" * 27+" { Script Exec Params } "+"-" * 27 +"\n" +" " * 30 +"+"+"-" * 18 +"+\n";
values = "\n\t" + str(main.params)
values = re.sub(",", "\n\t", values)
@@ -110,6 +110,7 @@
main.LogFileName = main.logdir + "/" + main.TEST + "_" +str(currentTime) + ".log"
main.ReportFileName = main.logdir + "/" + main.TEST + "_" + str(currentTime) + ".rpt"
+ main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
#### Add log-level - Report
logging.addLevelName(9, "REPORT")
@@ -228,7 +229,17 @@
#main.log.report(testResult)
main.testResult = testResult
main.log.exact(testResult)
-
+
+ ##CSV output needed for Jenkin's plot plugin
+ #NOTE: the elements were orded based on the colors assigned to the data
+ logfile = open(main.JenkinsCSV ,"w")
+ logfile.write(",".join( ['Tests Failed', 'Tests Passed', 'Tests Planned'] ) + "\n")
+ logfile.write(",".join( [str(main.TOTAL_TC_FAIL), str(main.TOTAL_TC_PASS), str(main.TOTAL_TC_PLANNED)] ))
+ logfile.close()
+
+
+
+
def updateCaseResults(self,main):
'''
Update the case result based on the steps execution and asserting each step in the test-case
diff --git a/TestON/dependencies/Jenkins_getresult_andrew.py b/TestON/dependencies/Jenkins_getresult_andrew.py
new file mode 100755
index 0000000..0e7ef8d
--- /dev/null
+++ b/TestON/dependencies/Jenkins_getresult_andrew.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import datetime
+import time
+import argparse
+
+parser = argparse.ArgumentParser()
+parser.add_argument("-n", "--name", help="Comma Separated string of test names. Ex: --name='test1, test2, test3'")
+args = parser.parse_args()
+
+#Pass in test names as a comma separated string argument.
+#Example: ./Jenkins_getresult.py "Test1,Test2,Test3,Test4"
+name_list = args.name.split(",")
+result_list = map(lambda x: x.strip(), name_list)
+
+#NOTE: testnames list should be in order in which it is run
+testnames = result_list
+output = ''
+testdate = datetime.datetime.now()
+
+output +="<p>**************************************</p>"
+output +=testdate.strftime('Jenkins test result for %H:%M on %b %d, %Y. %Z')
+
+#TestON reporting
+for test in testnames:
+ name = os.popen("ls /home/admin/ONLabTest/TestON/logs/ -rt | grep %s | tail -1" % test).read().split()[0]
+ path = "/home/admin/ONLabTest/TestON/logs/" + name + "/"
+ output +="<p></p>"
+ #output +=" Date: %s, %s %s" % (name.split("_")[2], name.split("_")[1], name.split("_")[3]) + "<br>*******************<br>"
+ #Open the latest log folder
+ output += "<h2>Test "+str(test)+"</h2><p>************************************</p>"
+
+ f = open(path + name + ".rpt")
+
+ #Parse through each line of logs and look for specific strings to output to wiki.
+ #NOTE: with current implementation, you must specify which output to output to wiki by using
+ #main.log.report("") since it is looking for the [REPORT] tag in the logs
+ for line in f:
+ if re.search("Result summary for Testcase", line):
+ output += "<h3>"+str(line)+"</h3>"
+ #output += "<br>"
+ if re.search("\[REPORT\]", line):
+ line_split = line.split("] ")
+ #line string is split by bracket, and first two items (log tags) in list are omitted from output
+ #join is used to convert list to string
+ line_str = ''.join(line_split[2:])
+ output += "<p>"
+ output += line_str
+ output += "</p>"
+ if re.search("Result:", line):
+ output += "<p>"
+ output += line
+ output += "</p>"
+ f.close()
+
+ #*********************
+ #include any other phrase specific to case you would like to include in wiki here
+ if test == "IntentPerf":
+ output += "URL to Historical Performance results data: <a href='http://10.128.5.54/perf.html'>Perf Graph</a>"
+ #*********************
+print output
diff --git a/TestON/dependencies/rotate.sh b/TestON/dependencies/rotate.sh
new file mode 100755
index 0000000..235d138
--- /dev/null
+++ b/TestON/dependencies/rotate.sh
@@ -0,0 +1,54 @@
+#!/bin/bash
+
+
+# NOTE: Taken fnd modified from onos.sh
+# pack-rotate-log [packname] "[log-filenames]" [max rotations]
+# Note: [packname] and all the log-files specified by [log-filenames]
+# must reside in same dir
+# Example:
+# pack="/foo/bar/testlogs"
+# logfiles="/foo/bar/test1.log /foo/bar/test*.log"
+# pack-rotate-log $pack "$logfiles" 5
+# => testlogs.tar.bz2 (will contain test1.log test2.log ...)
+# testlogs.tar.bz2 -> testlogs.1.tar.bz2
+# testlogs.1.tar.bz2 -> testlogs.2.tar.bz2
+# ...
+function pack-rotate-log {
+ local packname=$1
+ local logfiles=$2
+ local nr_max=${3:-10}
+ local suffix=".tar.bz2"
+
+ # rotate
+ for i in `seq $(expr $nr_max - 1) -1 1`; do
+ if [ -f ${packname}.${i}${suffix} ]; then
+ mv -f -- ${packname}.${i}${suffix} ${packname}.`expr $i + 1`${suffix}
+ fi
+ done
+ if [ -f ${packname}${suffix} ]; then
+ mv -- ${packname}${suffix} ${packname}.1${suffix}
+ fi
+
+ # pack
+ local existing_logfiles=$( ls -1 $logfiles 2>/dev/null | xargs -n1 basename 2>/dev/null)
+ if [ ! -z "${existing_logfiles}" ]; then
+ tar cjf ${packname}${suffix} -C `dirname ${packname}` -- ${existing_logfiles}
+ for word in ${existing_logfiles}
+ do
+ rm -- `dirname ${packname}`/${word}
+ done
+ fi
+}
+
+
+
+#Begin script
+#NOTE: This seems to break the TestON summary since it mentions the testname
+#echo "Rotating logs for '${1}' test"
+base_name=$1
+root_dir="/home/admin/packet_captures"
+timestamp=`date +%Y_%B_%d_%H_%M_%S`
+#Maybe this should be an argument? pack-and-rotate supports that
+nr_max=20
+
+pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap" ${nr_max}
diff --git a/TestON/drivers/common/cli/emulator/lincoedriver.py b/TestON/drivers/common/cli/emulator/lincoedriver.py
index 4d2cc01..6f7ce3a 100644
--- a/TestON/drivers/common/cli/emulator/lincoedriver.py
+++ b/TestON/drivers/common/cli/emulator/lincoedriver.py
@@ -293,8 +293,8 @@
'''
try:
#Send CTRL+C twice to exit CLI
- self.handle.sendline("\x03")
- self.handle.sendline("\x03")
+ self.handle.send("\x03")
+ self.handle.send("\x03")
self.handle.expect("\$")
except pexpect.EOF:
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 9191b89..2f31b7d 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -146,33 +146,44 @@
topoDict = self.num_switches_n_links(*topoArgList)
return topoDict
- def pingall(self):
+ def pingall(self, timeout=300):
'''
Verifies the reachability of the hosts using pingall command.
+ Optional parameter timeout allows you to specify how long to wait for pingall to complete
+ Returns:
+ main.TRUE if pingall completes with no pings dropped
+ otherwise main.FALSE
'''
if self.handle :
main.log.info(self.name+": Checking reachabilty to the hosts using pingall")
try:
- response = self.execute(cmd="pingall",prompt="mininet>",timeout=120)
- print "response: " + str(response)
- except pexpect.EOF:
+ response = self.execute(cmd="pingall",prompt="mininet>",timeout=int(timeout))
+ except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
- #main.cleanup()
- #main.exit()
- pattern = 'Results\:\s0\%\sdropped\s'
- #FIXME:Pending Mininet Pull Request #408
- #pattern = 'Results\:\s0\.00\%\sdropped\s'
- #if utilities.assert_matches(expect=pattern,actual=response,onpass="All hosts are reaching",onfail="Unable to reach all the hosts"):
+ main.cleanup()
+ main.exit()
+ except pexpect.TIMEOUT:
+ #We may not want to kill the test if pexpect times out
+ main.log.error(self.name + ": TIMEOUT exception found")
+ main.log.error(self.name + ": " + str(self.handle.before) )
+ #NOTE: mininet's pingall rounds, so we will check the number of passed and number of failed
+ pattern = "Results\:\s0\%\sdropped\s\((?P<passed>[\d]+)/(?P=passed)"
if re.search(pattern,response):
main.log.info(self.name+": All hosts are reachable")
return main.TRUE
else:
main.log.error(self.name+": Unable to reach all the hosts")
+ main.log.info("Pingall ouput: " + str(response))
+ #NOTE: Send ctrl-c to make sure pingall is done
+ self.handle.send("\x03")
+ self.handle.expect("Interrupt")
+ self.handle.expect("mininet>")
return main.FALSE
else :
main.log.error(self.name+": Connection failed to the host")
- return main.FALSE
+ main.cleanup()
+ main.exit()
def fpingHost(self,**pingParams):
'''
@@ -649,6 +660,187 @@
else:
main.log.info(response)
+ def add_switch( self, sw, **kwargs ):
+ '''
+ adds a switch to the mininet topology
+ NOTE: this uses a custom mn function
+ NOTE: cannot currently specify what type of switch
+ required params:
+ switchname = name of the new switch as a string
+ optional keyvalues:
+ dpid = "dpid"
+ returns: main.FASLE on an error, else main.TRUE
+ '''
+ dpid = kwargs.get('dpid', '')
+ command = "addswitch " + str( sw ) + " " + str( dpid )
+ try:
+ response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+ if re.search("already exists!", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("Error", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("usage:", response):
+ main.log.warn(response)
+ return main.FALSE
+ else:
+ return main.TRUE
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+
+ def del_switch( self, sw ):
+ '''
+ delete a switch from the mininet topology
+ NOTE: this uses a custom mn function
+ required params:
+ switchname = name of the switch as a string
+ returns: main.FASLE on an error, else main.TRUE
+ '''
+ command = "delswitch " + str( sw )
+ try:
+ response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+ if re.search("no switch named", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("Error", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("usage:", response):
+ main.log.warn(response)
+ return main.FALSE
+ else:
+ return main.TRUE
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+
+ def add_link( self, node1, node2 ):
+ '''
+ add a link to the mininet topology
+ NOTE: this uses a custom mn function
+ NOTE: cannot currently specify what type of link
+ required params:
+ node1 = the string node name of the first endpoint of the link
+ node2 = the string node name of the second endpoint of the link
+ returns: main.FASLE on an error, else main.TRUE
+ '''
+ command = "addlink " + str( node1 ) + " " + str( node2 )
+ try:
+ response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+ if re.search("doesnt exist!", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("Error", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("usage:", response):
+ main.log.warn(response)
+ return main.FALSE
+ else:
+ return main.TRUE
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+
+ def del_link( self, node1, node2 ):
+ '''
+ delete a link from the mininet topology
+ NOTE: this uses a custom mn function
+ required params:
+ node1 = the string node name of the first endpoint of the link
+ node2 = the string node name of the second endpoint of the link
+ returns: main.FASLE on an error, else main.TRUE
+ '''
+ command = "dellink " + str( node1 ) + " " + str( node2 )
+ try:
+ response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+ if re.search("no node named", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("Error", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("usage:", response):
+ main.log.warn(response)
+ return main.FALSE
+ else:
+ return main.TRUE
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+
+ def add_host( self, hostname, **kwargs ):
+ '''
+ Add a host to the mininet topology
+ NOTE: this uses a custom mn function
+ NOTE: cannot currently specify what type of host
+ required params:
+ hostname = the string hostname
+ optional key-value params
+ switch = "switch name"
+ returns: main.FASLE on an error, else main.TRUE
+ '''
+ switch = kwargs.get('switch', '')
+ command = "addhost " + str( hostname ) + " " + str( switch )
+ try:
+ response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+ if re.search("already exists!", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("doesnt exists!", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("Error", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("usage:", response):
+ main.log.warn(response)
+ return main.FALSE
+ else:
+ return main.TRUE
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+
+ def del_host( self, hostname ):
+ '''
+ delete a host from the mininet topology
+ NOTE: this uses a custom mn function
+ required params:
+ hostname = the string hostname
+ returns: main.FASLE on an error, else main.TRUE
+ '''
+ command = "delhost " + str( hostname )
+ try:
+ response = self.execute(cmd=command,prompt="mininet>",timeout=10)
+ if re.search("no host named", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("Error", response):
+ main.log.warn(response)
+ return main.FALSE
+ elif re.search("usage:", response):
+ main.log.warn(response)
+ return main.FALSE
+ else:
+ return main.TRUE
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
def disconnect(self):
main.log.info(self.name+": Disconnecting mininet...")
@@ -735,9 +927,10 @@
self.handle.expect("mininet>")
self.handle.sendline("sh sudo tcpdump -n -i "+ intf + " " + port + " -w " + filename.strip() + " &")
self.handle.sendline("")
- self.handle.sendline("")
i=self.handle.expect(['No\ssuch\device','listening\son',pexpect.TIMEOUT,"mininet>"],timeout=10)
main.log.warn(self.handle.before + self.handle.after)
+ self.handle.sendline("")
+ self.handle.expect("mininet>")
if i == 0:
main.log.error(self.name + ": tcpdump - No such device exists. tcpdump attempted on: " + intf)
return main.FALSE
@@ -769,7 +962,7 @@
"pkills tcpdump"
try:
self.handle.sendline("sh sudo pkill tcpdump")
- self.handle.sendline("")
+ self.handle.expect("mininet>")
self.handle.sendline("")
self.handle.expect("mininet>")
except pexpect.EOF:
@@ -857,7 +1050,7 @@
#FIXME: this does not look for extra ports in ONOS, only checks that ONOS has what is in MN
import json
from numpy import uint64
- port_results = main.TRUE
+ ports_results = main.TRUE
output = {"switches":[]}
for switch in topo.graph.switches: #iterate through the MN topology and pull out switches and and port info
ports = []
@@ -882,9 +1075,12 @@
for mn_switch in output['switches']:
mn_ports = []
onos_ports = []
+ switch_result = main.TRUE
for port in mn_switch['ports']:
if port['enabled'] == True:
mn_ports.append(port['of_port'])
+ #else: #DEBUG only
+ # main.log.warn("Port %s on switch %s is down" % ( str(port['of_port']) , str(mn_switch['name'])) )
for onos_switch in ports_json:
#print "Iterating through a new switch as seen by ONOS"
#print onos_switch
@@ -894,37 +1090,55 @@
if port['isEnabled']:
#print "Iterating through available ports on the switch"
#print port
- onos_ports.append(int(port['port']))
+ if port['port'] == 'local':
+ #onos_ports.append('local')
+ onos_ports.append(long(uint64(-2)))
+ else:
+ onos_ports.append(int(port['port']))
+ '''
+ else: #This is likely a new reserved port implemented
+ main.log.error("unkown port '" + str(port['port']) )
+ '''
+ #else: #DEBUG
+ # main.log.warn("Port %s on switch %s is down" % ( str(port['port']) , str(onos_switch['device']['id'])) )
+ break
mn_ports.sort(key=float)
onos_ports.sort(key=float)
#print "\nPorts for Switch %s:" % (switch['name'])
#print "\tmn_ports[] = ", mn_ports
#print "\tonos_ports[] = ", onos_ports
-
- #NOTE:For OF1.3, the OFP_local port number is 0xfffffffe or 4294967294 instead of 0xfffe or 65534 in OF1.0,
- # ONOS topology sees the correct port number, however MN topology as read from line 151 of
- # https://github.com/ucb-sts/sts/blob/topology_refactoring2/sts/entities/teston_entities.py
- # is 0xfffe which doesn't work correctly with OF1.3 switches.
-
- #NOTE: ONOS is abstracting port numbers to 64bit unsigned number(long). So we will be converting the
- # OF reserved ports to these numbers
-
+ mn_ports_log = mn_ports
+ onos_ports_log = onos_ports
+ mn_ports = [x for x in mn_ports]
+ onos_ports = [x for x in onos_ports]
#TODO: handle other reserved port numbers besides LOCAL
- for mn_port,onos_port in zip(mn_ports,onos_ports):
- #print "mn == onos port?"
- #print mn_port, onos_port
- if mn_port == onos_port or (mn_port == 65534 and onos_port == long(uint64(-2))):
- continue
+ #NOTE: Reserved ports
+ # Local port: -2 in Openflow, ONOS shows 'local', we store as long(uint64(-2))
+ for mn_port in mn_ports_log:
+ if mn_port in onos_ports:
#don't set results to true here as this is just one of many checks and it might override a failure
- else: #the ports of this switch don't match
- port_results = main.FALSE
- break
- if port_results == main.FALSE:
+ mn_ports.remove(mn_port)
+ onos_ports.remove(mn_port)
+ #NOTE: OVS reports this as down since there is no link
+ # So ignoring these for now
+ #TODO: Come up with a better way of handling these
+ if 65534 in mn_ports:
+ mn_ports.remove(65534)
+ if long(uint64(-2)) in onos_ports:
+ onos_ports.remove( long(uint64(-2)) )
+ if len(mn_ports): #the ports of this switch don't match
+ switch_result = main.FALSE
+ main.log.warn("Ports in MN but not ONOS: " + str(mn_ports) )
+ if len(onos_ports): #the ports of this switch don't match
+ switch_result = main.FALSE
+ main.log.warn("Ports in ONOS but not MN: " + str(onos_ports) )
+ if switch_result == main.FALSE:
main.log.report("The list of ports for switch %s(%s) does not match:" % (mn_switch['name'], mn_switch['dpid']) )
- main.log.report("mn_ports[] = " + str(mn_ports))
- main.log.report("onos_ports[] = " + str(onos_ports))
- return port_results
+ main.log.warn("mn_ports[] = " + str(mn_ports_log))
+ main.log.warn("onos_ports[] = " + str(onos_ports_log))
+ ports_results = ports_results and switch_result
+ return ports_results
@@ -1007,7 +1221,7 @@
if int(onos_port1) == int(port1) and int(onos_port2) == int(port2):
first_dir = main.TRUE
else:
- main.log.report('The port numbers do not match for ' +str(link) +\
+ main.log.warn('The port numbers do not match for ' +str(link) +\
' between ONOS and MN. When cheking ONOS for link '+\
'%s/%s -> %s/%s' % (node1, port1, node2, port2)+\
' ONOS has the values %s/%s -> %s/%s' %\
@@ -1018,7 +1232,7 @@
if ( int(onos_port1) == int(port2) and int(onos_port2) == int(port1) ):
second_dir = main.TRUE
else:
- main.log.report('The port numbers do not match for ' +str(link) +\
+ main.log.warn('The port numbers do not match for ' +str(link) +\
' between ONOS and MN. When cheking ONOS for link '+\
'%s/%s -> %s/%s' % (node2, port2, node1, port1)+\
' ONOS has the values %s/%s -> %s/%s' %\
@@ -1065,18 +1279,23 @@
'''
#TODO: Add error checking. currently the mininet command has no output
main.log.info("Updateing MN port information")
- self.handle.sendline("")
- self.handle.expect("mininet>")
-
- self.handle.sendline("update")
- self.handle.expect("mininet>")
+ try:
+ self.handle.sendline("")
+ self.handle.expect("mininet>")
- self.handle.sendline("")
- self.handle.expect("mininet>")
+ self.handle.sendline("update")
+ self.handle.expect("update")
+ self.handle.expect("mininet>")
- return main.TRUE
+ self.handle.sendline("")
+ self.handle.expect("mininet>")
-
+ return main.TRUE
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
if __name__ != "__main__":
import sys
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index bb03c07..1795a16 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -80,9 +80,16 @@
Returns main.ERROR if "found multiple mininet" is found and
Returns main.TRUE else
'''
+ #TODO: maybe we want to return the % loss instead? This way we can set an acceptible loss %.
+ #EX: 393 packets transmitted, 380 received, 3% packet loss, time 78519ms
+ # we may need to return a float to get around rounding errors
+
import os
self.handle.sendline("")
self.handle.expect("\$")
+ #Clear any output waiting in the bg from killing pings
+ self.handle.sendline("")
+ self.handle.expect("\$")
self.handle.sendline("cat " + pingList)
self.handle.expect(pingList)
self.handle.expect("\$")
@@ -92,7 +99,8 @@
elif re.search("found multiple mininet",outputs):
return main.ERROR
else:
- print outputs
+ main.log.error("Error, unexpected output in the ping file")
+ main.log.warn( outputs )
return main.TRUE
@@ -155,9 +163,19 @@
main.log.info( "Transferring ping files to TestStation" )
command = "scp /tmp/ping.* "+ str(testONUser) + "@" + str(testONIP) + ":/tmp/"
self.execute(cmd=command,prompt="100%",timeout=20)
+ #Make sure the output is cleared
self.handle.sendline("")
self.handle.expect("\$")
- return main.TRUE
+ self.handle.sendline("")
+ self.handle.expect("\$")
+ self.handle.sendline("")
+ i=self.handle.expect(["password","\$"])
+ if i == 0:
+ main.log.error("Error, sudo asking for password")
+ main.log.error(self.handle.before)
+ return main.FALSE
+ else:
+ return main.TRUE
def pingLongKill(self):
import time
@@ -209,34 +227,6 @@
main.last_result = main.FALSE
return main.FALSE
-
- def pingall(self):
- '''
- Verifies the reachability of the hosts using pingall command.
- This function is required by Packey Optical test
- '''
- if self.handle :
- main.log.info(self.name+": Checking reachabilty to the hosts using pingall")
- try:
- response = self.execute(cmd="pingall",prompt="mininet>",timeout=120)
- print "response: " + str(response)
- except pexpect.EOF:
- main.log.error(self.name + ": EOF exception found")
- main.log.error(self.name + ": " + self.handle.before)
- pattern = 'Results\:\s0\%\sdropped\s'
- if re.search(pattern,response):
- main.log.info(self.name+": All hosts are reachable")
- return main.TRUE
- else:
- main.log.error(self.name+": Unable to reach all the hosts")
- return main.FALSE
- else :
- main.log.error(self.name+": Connection failed to the host")
- return main.FALSE
-
-
-
-
def pingHost(self,**pingParams):
'''
Pings between two hosts on remote mininet
@@ -286,154 +276,14 @@
else :
main.log.error("Connection failed to the host")
- def ctrl_none(self):
- '''
- Sets all the switches to no controllers.
- '''
- self.execute(cmd="~/ONOS/scripts/test-ctrl-none.sh", prompt="\$",timeout=10)
-
- def ctrl_one(self, ip):
- '''
- Sets all the switches to point to the supplied IP
- '''
- self.execute(cmd="~/ONOS/scripts/test-ctrl-one.sh "+ip, prompt="\$",timeout=10)
-
- def ctrl_local(self):
- '''
- Sets all the switches to point to the Controller on the same machine that they are running on.
- '''
- self.execute(cmd="~/ONOS/scripts/test-ctrl-local.sh ", prompt="\$",timeout=10)
-
- # def verifySSH(self,**connectargs):
- # response = self.execute(cmd="h1 /usr/sbin/sshd -D&",prompt="mininet>",timeout=10)
- # response = self.execute(cmd="h4 /usr/sbin/sshd -D&",prompt="mininet>",timeout=10)
- # for key in connectargs:
- # vars(self)[key] = connectargs[key]
- # response = self.execute(cmd="xterm h1 h4 ",prompt="mininet>",timeout=10)
- # import time
- # time.sleep(20)
- # if self.flag == 0:
- # self.flag = 1
- # return main.FALSE
- # else :
- # return main.TRUE
- #
- # def getMacAddress(self,host):
- # '''
- # Verifies the host's ip configured or not.
- # '''
- # if self.handle :
- # response = self.execute(cmd=host+" ifconfig",prompt="mininet>",timeout=10)
-
- # pattern = "HWaddr\s(((\d|\w)+:)+(\d|\w))"
- # mac_address_search = re.search(pattern, response)
- # main.log.info("Mac-Address of Host "+host +" is "+mac_address_search.group(1))
- # return mac_address_search.group(1)
- # else :
- # main.log.error("Connection failed to the host")
- # def getIPAddress(self,host):
- # '''
- # Verifies the host's ip configured or not.
- # '''
- # if self.handle :
- # response = self.execute(cmd=host+" ifconfig",prompt="mininet>",timeout=10)
-
- # pattern = "inet\saddr:(\d+\.\d+\.\d+\.\d+)"
- # ip_address_search = re.search(pattern, response)
- # main.log.info("IP-Address of Host "+host +" is "+ip_address_search.group(1))
- # return ip_address_search.group(1)
- # else :
- # main.log.error("Connection failed to the host")
- #
- # def dump(self):
- # main.log.info("Dump node info")
- # self.execute(cmd = 'dump',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def intfs(self):
- # main.log.info("List interfaces")
- # self.execute(cmd = 'intfs',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def net(self):
- # main.log.info("List network connections")
- # self.execute(cmd = 'net',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def iperf(self):
- # main.log.info("Simple iperf TCP test between two (optionally specified) hosts")
- # self.execute(cmd = 'iperf',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def iperfudp(self):
- # main.log.info("Simple iperf TCP test between two (optionally specified) hosts")
- # self.execute(cmd = 'iperfudp',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def nodes(self):
- # main.log.info("List all nodes.")
- # self.execute(cmd = 'nodes',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def pingpair(self):
- # main.log.infoe("Ping between first two hosts")
- # self.execute(cmd = 'pingpair',prompt = 'mininet>',timeout = 20)
- #
- # if utilities.assert_matches(expect='0% packet loss',actual=response,onpass="No Packet loss",onfail="Hosts not reachable"):
- # main.log.info("Ping between two hosts SUCCESS")
- # main.last_result = main.TRUE
- # return main.TRUE
- # else :
- # main.log.error("PACKET LOST, HOSTS NOT REACHABLE")
- # main.last_result = main.FALSE
- # return main.FALSE
- #
- # def link(self,**linkargs):
- # '''
- # Bring link(s) between two nodes up or down
- # '''
- # main.log.info('Bring link(s) between two nodes up or down')
- # args = utilities.parse_args(["END1","END2","OPTION"],**linkargs)
- # end1 = args["END1"] if args["END1"] != None else ""
- # end2 = args["END2"] if args["END2"] != None else ""
- # option = args["OPTION"] if args["OPTION"] != None else ""
- # command = "link "+str(end1) + " " + str(end2)+ " " + str(option)
- # response = self.execute(cmd=command,prompt="mininet>",timeout=10)
- # return main.TRUE
- #
-
- # def dpctl(self,**dpctlargs):
- # '''
- # Run dpctl command on all switches.
- # '''
- # main.log.info('Run dpctl command on all switches')
- # args = utilities.parse_args(["CMD","ARGS"],**dpctlargs)
- # cmd = args["CMD"] if args["CMD"] != None else ""
- # cmdargs = args["ARGS"] if args["ARGS"] != None else ""
- # command = "dpctl "+cmd + " " + str(cmdargs)
- # response = self.execute(cmd=command,prompt="mininet>",timeout=10)
- # return main.TRUE
- #
- #
- # def get_version(self):
- # file_input = path+'/lib/Mininet/INSTALL'
- # version = super(Mininet, self).get_version()
- # pattern = 'Mininet\s\w\.\w\.\w\w*'
- # for line in open(file_input,'r').readlines():
- # result = re.match(pattern, line)
- # if result:
- # version = result.group(0)
- #
- #
- # return version
- def start_tcpdump(self, filename, intf = "eth0", port = "port 6633"):
+ def start_tcpdump(self, filename, intf = "eth0", port = "port 6633", user="admin"):
'''
Runs tpdump on an intferface and saves the file
intf can be specified, or the default eth0 is used
'''
try:
self.handle.sendline("")
- self.handle.sendline("sudo tcpdump -n -i "+ intf + " " + port + " -w " + filename.strip() + " &")
+ self.handle.sendline("sudo tcpdump -n -i "+ intf + " " + port + " -w " + filename.strip() + " -Z " + user + " &")
self.handle.sendline("")
self.handle.sendline("")
i=self.handle.expect(['No\ssuch\device','listening\son',pexpect.TIMEOUT,"\$"],timeout=10)
@@ -502,49 +352,6 @@
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
return main.FALSE
-
- def del_switch(self,sw):
- self.handle.sendline("")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl del-br "+sw)
- self.handle.expect("\$")
- return main.TRUE
-
- def add_switch(self,sw):
- #FIXME: Remove hardcoded number of ports
- self.handle.sendline("")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-br "+sw)
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth1")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth2")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth3")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth4")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth5")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth6")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth7")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth8")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth9")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth10")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth11")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth12")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth13")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth14")
- self.handle.expect("\$")
-
def disconnect(self):
'''
@@ -553,7 +360,7 @@
response = ''
#print "Disconnecting Mininet"
if self.handle:
- self.handle.sendline("exit")
+ self.handle.sendline("exit")
self.handle.expect("exit")
self.handle.expect("(.*)")
response = self.handle.before
@@ -561,13 +368,13 @@
else :
main.log.error("Connection failed to the host")
response = main.FALSE
- return response
+ return response
def get_flowTable(self, protoVersion, sw):
+ #TODO document usage
+ #FIXME: clean up print statements and such
self.handle.sendline("cd")
- #self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
- print "cd expect status: "
- print self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
+ self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
#TODO: Write seperate versions of the function for this, possibly a string that tells it which switch is in use?
#For 1.0 version of OVS
#command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $6 $7 }' |sort -n -k1"
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 282200d..4916ba5 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -90,7 +90,7 @@
self.handle.expect("Confirm")
self.handle.sendline("yes")
self.handle.expect("\$")
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline("exit")
self.handle.expect("closed")
@@ -194,11 +194,13 @@
else:
#If failed, send ctrl+c to process and try again
main.log.info("Starting CLI failed. Retrying...")
- self.handle.sendline("\x03")
+ self.handle.send("\x03")
self.handle.sendline("onos -w "+str(ONOS_ip))
i = self.handle.expect(["onos>",pexpect.TIMEOUT],
timeout=30)
if i == 0:
+ main.log.info(str(ONOS_ip)+" CLI Started "+
+ "successfully after retry attempt")
return main.TRUE
else:
main.log.error("Connection to CLI "+\
@@ -459,27 +461,21 @@
main.log.info(self.name+" ::::::")
main.cleanup()
main.exit()
-
- def devices(self, json_format=True, grep_str=""):
+
+ def devices(self, json_format=True):
'''
Lists all infrastructure devices or switches
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
if json_format:
- if not grep_str:
- self.handle.sendline("devices -j")
- self.handle.expect("devices -j")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("devices -j | grep '"+
- str(grep_str)+"'")
- self.handle.expect("devices -j | grep '"+str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("devices -j")
+ self.handle.expect("devices -j")
+ self.handle.expect("onos>")
handle = self.handle.before
'''
handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
@@ -495,13 +491,8 @@
#print "repr(handle1) = ", repr(handle1)
return handle1
else:
- if not grep_str:
- self.handle.sendline("devices")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("devices | grep '"+
- str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("devices")
+ self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
return handle
@@ -517,34 +508,28 @@
main.cleanup()
main.exit()
- def links(self, json_format=True, grep_str=""):
+ def links(self, json_format=True):
'''
Lists all core links
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
if json_format:
- if not grep_str:
- self.handle.sendline("links -j")
- self.handle.expect("links -j")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("links -j | grep '"+
- str(grep_str)+"'")
- self.handle.expect("links -j | grep '"+str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("links -j")
+ self.handle.expect("links -j")
+ self.handle.expect("onos>")
handle = self.handle.before
'''
handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
- So we take off that escape sequence using
+ So we take off that escape sequence using
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
- handle1 = ansi_escape.sub('', handle)
+ handle1 = ansi_escape.sub('', handle)
'''
#print "repr(handle) =", repr(handle)
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -552,13 +537,8 @@
#print "repr(handle1) = ", repr(handle1)
return handle1
else:
- if not grep_str:
- self.handle.sendline("links")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("links | grep '"+
- str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("links")
+ self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
return handle
@@ -575,34 +555,28 @@
main.exit()
- def ports(self, json_format=True, grep_str=""):
+ def ports(self, json_format=True):
'''
Lists all ports
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
if json_format:
- if not grep_str:
- self.handle.sendline("ports -j")
- self.handle.expect("ports -j")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("ports -j | grep '"+
- str(grep_str)+"'")
- self.handle.expect("ports -j | grep '"+str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("ports -j")
+ self.handle.expect("ports -j")
+ self.handle.expect("onos>")
handle = self.handle.before
'''
handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
- So we take off that escape sequence using the following commads:
+ So we take off that escape sequence using the following commads:
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
- handle1 = ansi_escape.sub('', handle)
+ handle1 = ansi_escape.sub('', handle)
'''
#print "repr(handle) =", repr(handle)
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -611,20 +585,13 @@
return handle1
else:
- if not grep_str:
- self.handle.sendline("ports")
- self.handle.expect("onos>")
- self.handle.sendline("")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("ports | grep '"+
- str(grep_str)+"'")
- self.handle.expect("onos>")
- self.handle.sendline("")
- self.handle.expect("onos>")
+ self.handle.sendline("ports")
+ self.handle.expect("onos>")
+ self.handle.sendline("")
+ self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
- return handle
+ return handle
except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
@@ -638,34 +605,32 @@
main.exit()
- def roles(self, json_format=True, grep_str=""):
+ def roles(self, json_format=True):
'''
Lists all devices and the controllers with roles assigned to them
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
if json_format:
- if not grep_str:
- self.handle.sendline("roles -j")
- self.handle.expect("roles -j")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("roles -j | grep '"+
- str(grep_str)+"'")
- self.handle.expect("roles -j | grep '"+str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("roles -j")
+ self.handle.expect("roles -j")
+ self.handle.expect("onos>")
handle = self.handle.before
'''
- handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
- To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
- In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
- So we take off that escape sequence using the following commads:
+ handle variable here contains some ANSI escape color code sequences at the
+ end which are invisible in the print command output. To make that escape
+ sequence visible, use repr() function. The repr(handle) output when printed
+ shows the ANSI escape sequences. In json.loads(somestring), this somestring
+ variable is actually repr(somestring) and json.loads would fail with the escape
+ sequence.
+
+ So we take off that escape sequence using the following commads:
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
- handle1 = ansi_escape.sub('', handle)
+ handle1 = ansi_escape.sub('', handle)
'''
#print "repr(handle) =", repr(handle)
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -674,20 +639,13 @@
return handle1
else:
- if not grep_str:
- self.handle.sendline("roles")
- self.handle.expect("onos>")
- self.handle.sendline("")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("roles | grep '"+
- str(grep_str)+"'")
- self.handle.expect("onos>")
- self.handle.sendline("")
- self.handle.expect("onos>")
+ self.handle.sendline("roles")
+ self.handle.expect("onos>")
+ self.handle.sendline("")
+ self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
- return handle
+ return handle
except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
@@ -775,35 +733,29 @@
main.log.info(self.name+" ::::::")
main.cleanup()
main.exit()
-
- def hosts(self, json_format=True, grep_str=""):
+
+ def hosts(self, json_format=True):
'''
- Lists all discovered hosts
+ Lists all discovered hosts
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
if json_format:
- if not grep_str:
- self.handle.sendline("hosts -j")
- self.handle.expect("hosts -j")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("hosts -j | grep '"+
- str(grep_str)+"'")
- self.handle.expect("hosts -j | grep '"+str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("hosts -j")
+ self.handle.expect("hosts -j")
+ self.handle.expect("onos>")
handle = self.handle.before
'''
handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
- So we take off that escape sequence using
+ So we take off that escape sequence using
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
- handle1 = ansi_escape.sub('', handle)
+ handle1 = ansi_escape.sub('', handle)
'''
#print "repr(handle) =", repr(handle)
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -811,13 +763,8 @@
#print "repr(handle1) = ", repr(handle1)
return handle1
else:
- if not grep_str:
- self.handle.sendline("hosts")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("hosts | grep '"+
- str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("hosts")
+ self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
return handle
@@ -915,24 +862,24 @@
* host_id_two: ONOS host id for host2
Description:
Adds a host-to-host intent (bidrectional) by
- specifying the two hosts.
+ specifying the two hosts.
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
self.handle.sendline("add-host-intent "+
str(host_id_one) + " " + str(host_id_two))
self.handle.expect("onos>")
handle = self.handle.before
- print "handle =", handle
+ #print "handle =", handle
main.log.info("Intent installed between "+
str(host_id_one) + " and " + str(host_id_two))
return handle
-
+
except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
@@ -981,8 +928,8 @@
main.cleanup()
main.exit()
- def add_point_intent(self, ingress_device, port_ingress,
- egress_device, port_egress, ethType="", ethSrc="",
+ def add_point_intent(self, ingress_device, egress_device,
+ port_ingress="", port_egress="", ethType="", ethSrc="",
ethDst="", bandwidth="", lambda_alloc=False,
ipProto="", ipSrc="", ipDst="", tcpSrc="", tcpDst=""):
'''
@@ -1017,12 +964,11 @@
and not bandwidth and not lambda_alloc \
and not ipProto and not ipSrc and not ipDst \
and not tcpSrc and not tcpDst:
- cmd = "add-point-intent "+\
- str(ingress_device) + "/" + str(port_ingress) + " " +\
- str(egress_device) + "/" + str(port_egress)
-
+ cmd = "add-point-intent"
+
+
else:
- cmd = "add-point-intent "
+ cmd = "add-point-intent"
if ethType:
cmd += " --ethType " + str(ethType)
@@ -1045,18 +991,39 @@
if tcpDst:
cmd += " --tcpDst " + str(tcpDst)
- cmd += " "+str(ingress_device) +\
- "/" + str(port_ingress) + " " +\
- str(egress_device) + "/" + str(port_egress)
+ #Check whether the user appended the port
+ #or provided it as an input
+ if "/" in ingress_device:
+ cmd += " "+str(ingress_device)
+ else:
+ if not port_ingress:
+ main.log.error("You must specify "+
+ "the ingress port")
+ #TODO: perhaps more meaningful return
+ return main.FALSE
+
+ cmd += " "+ \
+ str(ingress_device) + "/" +\
+ str(port_ingress) + " "
+
+ if "/" in egress_device:
+ cmd += " "+str(egress_device)
+ else:
+ if not port_egress:
+ main.log.error("You must specify "+
+ "the egress port")
+ return main.FALSE
+
+ cmd += " "+\
+ str(egress_device) + "/" +\
+ str(port_egress)
self.handle.sendline(cmd)
+
+ main.log.info(cmd + " sent")
i = self.handle.expect([
"Error",
"onos>"])
-
- self.handle.sendline("intents")
- self.handle.expect("onos>")
- Intenthandle = self.handle.before
if i == 0:
main.log.error("Error in adding point-to-point intent")
@@ -1149,7 +1116,7 @@
main.cleanup()
main.exit()
- def intents(self, json_format = False):
+ def intents(self, json_format = True):
'''
Optional:
* json_format: enable output formatting in json
@@ -1187,7 +1154,7 @@
main.cleanup()
main.exit()
- def flows(self, json_format = False):
+ def flows(self, json_format = True):
'''
Optional:
* json_format: enable output formatting in json
@@ -1200,6 +1167,8 @@
self.handle.expect("flows -j")
self.handle.expect("onos>")
handle = self.handle.before
+ ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
+ handle = ansi_escape.sub('', handle)
else:
self.handle.sendline("")
@@ -1207,6 +1176,8 @@
self.handle.sendline("flows")
self.handle.expect("onos>")
handle = self.handle.before
+ if re.search("Error\sexecuting\scommand:", handle):
+ main.log.error(self.name + ".flows() response: " + str(handle))
return handle
@@ -1222,6 +1193,51 @@
main.cleanup()
main.exit()
+ def push_test_intents(self, dpid_src, dpid_dst, num_intents,
+ report=True):
+ '''
+ Description:
+ Push a number of intents in a batch format to
+ a specific point-to-point intent definition
+ Required:
+ * dpid_src: specify source dpid
+ * dpid_dst: specify destination dpid
+ * num_intents: specify number of intents to push
+ Optional:
+ * report: default True, returns latency information
+ '''
+ try:
+ cmd = "push-test-intents "+\
+ str(dpid_src)+" "+str(dpid_dst)+" "+\
+ str(num_intents)
+ self.handle.sendline(cmd)
+ self.handle.expect(cmd)
+ self.handle.expect("onos>")
+
+ handle = self.handle.before
+
+ #Some color thing that we want to escape
+ ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
+ handle = ansi_escape.sub('', handle)
+
+ if report:
+ main.log.info(handle)
+ return handle
+ else:
+ return main.TRUE
+
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
+
def intents_events_metrics(self, json_format=True):
'''
Description:Returns topology metrics
@@ -1578,5 +1594,54 @@
main.cleanup()
main.exit()
+ def clusters(self, json_format=True):
+ '''
+ Lists all clusters
+ Optional argument:
+ * json_format - boolean indicating if you want output in json
+ '''
+ try:
+ self.handle.sendline("")
+ self.handle.expect("onos>")
+
+ if json_format:
+ self.handle.sendline("clusters -j")
+ self.handle.expect("clusters -j")
+ self.handle.expect("onos>")
+ handle = self.handle.before
+ '''
+ handle variable here contains some ANSI escape color code
+ sequences at the end which are invisible in the print command
+ output. To make that escape sequence visible, use repr() function.
+ The repr(handle) output when printed shows the ANSI escape sequences.
+ In json.loads(somestring), this somestring variable is actually
+ repr(somestring) and json.loads would fail with the escape sequence.
+ So we take off that escape sequence using
+ ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
+ handle1 = ansi_escape.sub('', handle)
+ '''
+ #print "repr(handle) =", repr(handle)
+ ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
+ handle1 = ansi_escape.sub('', handle)
+ #print "repr(handle1) = ", repr(handle1)
+ return handle1
+ else:
+ self.handle.sendline("clusters")
+ self.handle.expect("onos>")
+ handle = self.handle.before
+ #print "handle =",handle
+ return handle
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
+
#***********************************
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 5cc7eb6..e2625cf 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -82,7 +82,7 @@
'''
response = ''
try:
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline("exit")
self.handle.expect("closed")
@@ -103,7 +103,7 @@
try:
self.handle.sendline("onos-package")
self.handle.expect("onos-package")
- self.handle.expect("tar.gz",timeout=10)
+ self.handle.expect("tar.gz",timeout=30)
handle = str(self.handle.before)
main.log.info("onos-package command returned: "+
handle)
@@ -164,7 +164,7 @@
self.handle.sendline("cd "+ self.home)
self.handle.expect("\$")
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline("mvn clean install")
self.handle.expect("mvn clean install")
@@ -195,7 +195,7 @@
for line in self.handle.before.splitlines():
if "Total time:" in line:
main.log.info(line)
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$", timeout=60)
return main.TRUE
elif i == 4:
@@ -392,10 +392,11 @@
self.handle.sendline("export TERM=xterm-256color")
self.handle.expect("xterm-256color")
self.handle.expect("\$")
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline("cd " + self.home + "; git log -1 --pretty=fuller --decorate=short | grep -A 6 \"commit\" --color=never")
- self.handle.expect("--color=never")
+ #NOTE: for some reason there are backspaces inserted in this phrase when run from Jenkins on some tests
+ self.handle.expect("never")
self.handle.expect("\$")
response=(self.name +": \n"+ str(self.handle.before + self.handle.after))
self.handle.sendline("cd " + self.home)
@@ -405,13 +406,23 @@
print line
if report:
for line in lines[2:-1]:
- main.log.report(line)
+ #Bracket replacement is for Wiki-compliant
+ #formatting. '<' or '>' are interpreted
+ #as xml specific tags that cause errors
+ line = line.replace("<","[")
+ line = line.replace(">","]")
+ main.log.report("\t" + line)
return lines[2]
except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
main.cleanup()
main.exit()
+ except pexpect.TIMEOUT:
+ main.log.error(self.name + ": TIMEOUT exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
except:
main.log.info(self.name + ":::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::")
main.log.error( traceback.print_exc() )
@@ -778,7 +789,7 @@
try:
self.handle.sendline("")
self.handle.expect("\$")
- self.handle.sendline("onos-uninstall "+str(node_ip))
+ self.handle.sendline( "onos-uninstall "+str(node_ip) )
self.handle.expect("\$")
main.log.info("ONOS "+node_ip+" was uninstalled")
@@ -844,18 +855,24 @@
def onos_remove_raft_logs(self):
'''
Removes Raft / Copy cat files from ONOS to ensure
- a cleaner environment.
-
+ a cleaner environment.
+
Description:
- Stops all ONOS defined in the cell,
+ Stops all ONOS defined in the cell,
wipes the raft / copycat log files
'''
try:
self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline("onos-remove-raft-logs")
- self.handle.expect("\$")
-
+ #Sometimes this command hangs
+ i = self.handle.expect(["\$", pexpect.TIMEOUT],
+ timeout=120)
+ if i == 1:
+ i = self.handle.expect(["\$", pexpect.TIMEOUT],
+ timeout=120)
+ if i == 1:
+ return main.FALSE
return main.TRUE
except pexpect.EOF:
@@ -869,7 +886,7 @@
main.log.info(self.name+" ::::::")
main.cleanup()
main.exit()
-
+
def onos_start_network(self, mntopo):
'''
Calls the command 'onos-start-network [<mininet-topo>]
@@ -921,14 +938,15 @@
self.handle.sendline("onos-wait-for-start " + node )
self.handle.expect("onos-wait-for-start")
#NOTE: this timeout is arbitrary"
- i = self.handle.expect(["\$", pexpect.TIMEOUT], timeout = 300)
+ i = self.handle.expect(["\$", pexpect.TIMEOUT], timeout = 120)
if i == 0:
main.log.info(self.name + ": " + node + " is up")
return main.TRUE
elif i == 1:
#NOTE: since this function won't return until ONOS is ready,
# we will kill it on timeout
- self.handle.sendline("\003") #Control-C
+ main.log.error("ONOS has not started yet")
+ self.handle.send("\x03") #Control-C
self.handle.expect("\$")
return main.FALSE
except pexpect.EOF:
@@ -1226,5 +1244,3 @@
main.log.error( traceback.print_exc())
main.log.info(self.name+" ::::::")
-
-
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 9ae90fb..7bf7667 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -76,7 +76,7 @@
elif i==6:
main.log.info("Password not required logged in")
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect('>|#|\$')
return self.handle
@@ -142,7 +142,7 @@
main.log.info("Found More screen to go , Sending a key to proceed")
indexMore = self.handle.expect(["^:$", expectPrompt], timeout = timeoutVar)
while indexMore == 0:
- main.log.info("Found anoother More screen to go , Sending a key to proceed")
+ main.log.info("Found another More screen to go , Sending a key to proceed")
self.handle.sendcontrol("D")
indexMore = self.handle.expect(["^:$", expectPrompt], timeout = timeoutVar)
self.LASTRSP = self.LASTRSP + self.handle.before
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
new file mode 100644
index 0000000..0a8bd57
--- /dev/null
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -0,0 +1,63 @@
+<PARAMS>
+ <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <ENV>
+ <cellName>HA</cellName>
+ </ENV>
+ <Git>False</Git>
+
+ <CTRL>
+ <ip1>10.128.30.11</ip1>
+ <port1>6633</port1>
+
+ <ip2>10.128.30.12</ip2>
+ <port2>6633</port2>
+
+ <ip3>10.128.30.13</ip3>
+ <port3>6633</port3>
+
+ <ip4>10.128.30.14</ip4>
+ <port4>6633</port4>
+
+ <ip5>10.128.30.15</ip5>
+ <port5>6633</port5>
+
+ <ip6>10.128.30.16</ip6>
+ <port6>6633</port6>
+
+ <ip7>10.128.30.17</ip7>
+ <port7>6633</port7>
+ </CTRL>
+ <TESTONUSER>admin</TESTONUSER>
+ <TESTONIP>10.128.30.10</TESTONIP>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+ <timers>
+ <LinkDiscovery>2</LinkDiscovery>
+ <SwitchDiscovery>2</SwitchDiscovery>
+ </timers>
+ <MNtcpdump>
+ <intf>eth0</intf>
+ <port> </port>
+ <folder>~/packet_captures/</folder>
+ </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
new file mode 100644
index 0000000..47dc744
--- /dev/null
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -0,0 +1,1202 @@
+'''
+Description: This test is to determine if ONOS can handle
+ all of it's nodes restarting
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE6: The Failure case.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+'''
+class HATestClusterRestart:
+
+ def __init__(self) :
+ self.default = ''
+
+ def CASE1(self,main) :
+ '''
+ CASE1 is to compile ONOS and push it to the test machines
+
+ Startup sequence:
+ git pull
+ mvn clean install
+ onos-package
+ cell <name>
+ onos-verify-cell
+ NOTE: temporary - onos-remove-raft-logs
+ onos-install -f
+ onos-wait-for-start
+ '''
+ import time
+ main.log.report("ONOS HA test: Restart all ONOS nodes - initialization")
+ main.case("Setting up test environment")
+
+ # load some vairables from the params file
+ PULL_CODE = False
+ if main.params['Git'] == 'True':
+ PULL_CODE = True
+ cell_name = main.params['ENV']['cellName']
+
+ #set global variables
+ global ONOS1_ip
+ global ONOS1_port
+ global ONOS2_ip
+ global ONOS2_port
+ global ONOS3_ip
+ global ONOS3_port
+ global ONOS4_ip
+ global ONOS4_port
+ global ONOS5_ip
+ global ONOS5_port
+ global ONOS6_ip
+ global ONOS6_port
+ global ONOS7_ip
+ global ONOS7_port
+
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS1_port = main.params['CTRL']['port1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS2_port = main.params['CTRL']['port2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS3_port = main.params['CTRL']['port3']
+ ONOS4_ip = main.params['CTRL']['ip4']
+ ONOS4_port = main.params['CTRL']['port4']
+ ONOS5_ip = main.params['CTRL']['ip5']
+ ONOS5_port = main.params['CTRL']['port5']
+ ONOS6_ip = main.params['CTRL']['ip6']
+ ONOS6_port = main.params['CTRL']['port6']
+ ONOS7_ip = main.params['CTRL']['ip7']
+ ONOS7_port = main.params['CTRL']['port7']
+
+
+ main.step("Applying cell variable to environment")
+ cell_result = main.ONOSbench.set_cell(cell_name)
+ verify_result = main.ONOSbench.verify_cell()
+
+ #FIXME:this is short term fix
+ main.log.report("Removing raft logs")
+ main.ONOSbench.onos_remove_raft_logs()
+ main.log.report("Uninstalling ONOS")
+ main.ONOSbench.onos_uninstall(ONOS1_ip)
+ main.ONOSbench.onos_uninstall(ONOS2_ip)
+ main.ONOSbench.onos_uninstall(ONOS3_ip)
+ main.ONOSbench.onos_uninstall(ONOS4_ip)
+ main.ONOSbench.onos_uninstall(ONOS5_ip)
+ main.ONOSbench.onos_uninstall(ONOS6_ip)
+ main.ONOSbench.onos_uninstall(ONOS7_ip)
+
+ clean_install_result = main.TRUE
+ git_pull_result = main.TRUE
+
+ main.step("Compiling the latest version of ONOS")
+ if PULL_CODE:
+ main.step("Git checkout and pull master")
+ main.ONOSbench.git_checkout("master")
+ git_pull_result = main.ONOSbench.git_pull()
+
+ main.step("Using mvn clean & install")
+ clean_install_result = main.TRUE
+ if git_pull_result == main.TRUE:
+ clean_install_result = main.ONOSbench.clean_install()
+ else:
+ main.log.warn("Did not pull new code so skipping mvn "+ \
+ "clean install")
+ main.ONOSbench.get_version(report=True)
+
+ main.step("Creating ONOS package")
+ package_result = main.ONOSbench.onos_package()
+
+ main.step("Installing ONOS package")
+ onos1_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS1_ip)
+ onos2_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS2_ip)
+ onos3_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS3_ip)
+ onos4_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS4_ip)
+ onos5_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS5_ip)
+ onos6_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS6_ip)
+ onos7_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS7_ip)
+ onos_install_result = onos1_install_result and onos2_install_result\
+ and onos3_install_result and onos4_install_result\
+ and onos5_install_result and onos6_install_result\
+ and onos7_install_result
+ '''
+ #FIXME: work around until onos is less fragile
+ main.ONOSbench.handle.sendline("onos-cluster-install")
+ print main.ONOSbench.handle.expect("\$")
+ onos_install_result = main.TRUE
+ '''
+
+
+ main.step("Checking if ONOS is up yet")
+ #TODO: Refactor
+ # check bundle:list?
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ if not onos2_isup:
+ main.log.report("ONOS2 didn't start!")
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ if not onos3_isup:
+ main.log.report("ONOS3 didn't start!")
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ if not onos4_isup:
+ main.log.report("ONOS4 didn't start!")
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ if not onos5_isup:
+ main.log.report("ONOS5 didn't start!")
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ if not onos6_isup:
+ main.log.report("ONOS6 didn't start!")
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ if not onos7_isup:
+ main.log.report("ONOS7 didn't start!")
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ # TODO: if it becomes an issue, we can retry this step a few times
+
+
+ cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+ cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+ cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+ cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
+ cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
+ cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
+ cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
+ cli_results = cli_result1 and cli_result2 and cli_result3 and\
+ cli_result4 and cli_result5 and cli_result6 and cli_result7
+
+ main.step("Start Packet Capture MN")
+ main.Mininet2.start_tcpdump(
+ str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+ intf = main.params['MNtcpdump']['intf'],
+ port = main.params['MNtcpdump']['port'])
+
+
+ case1_result = (clean_install_result and package_result and
+ cell_result and verify_result and onos_install_result and
+ onos_isup_result and cli_results)
+
+ utilities.assert_equals(expect=main.TRUE, actual=case1_result,
+ onpass="Test startup successful",
+ onfail="Test startup NOT successful")
+
+
+ #if case1_result==main.FALSE:
+ # main.cleanup()
+ # main.exit()
+
+ def CASE2(self,main) :
+ '''
+ Assign mastership to controllers
+ '''
+ import time
+ import json
+ import re
+
+
+ '''
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS1_port = main.params['CTRL']['port1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS2_port = main.params['CTRL']['port2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS3_port = main.params['CTRL']['port3']
+ ONOS4_ip = main.params['CTRL']['ip4']
+ ONOS4_port = main.params['CTRL']['port4']
+ ONOS5_ip = main.params['CTRL']['ip5']
+ ONOS5_port = main.params['CTRL']['port5']
+ ONOS6_ip = main.params['CTRL']['ip6']
+ ONOS6_port = main.params['CTRL']['port6']
+ ONOS7_ip = main.params['CTRL']['ip7']
+ ONOS7_port = main.params['CTRL']['port7']
+ '''
+
+
+ main.log.report("Assigning switches to controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign switches to controllers")
+
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+ ip1=ONOS1_ip,port1=ONOS1_port,
+ ip2=ONOS2_ip,port2=ONOS2_port,
+ ip3=ONOS3_ip,port3=ONOS3_port,
+ ip4=ONOS4_ip,port4=ONOS4_port,
+ ip5=ONOS5_ip,port5=ONOS5_port,
+ ip6=ONOS6_ip,port6=ONOS6_port,
+ ip7=ONOS7_ip,port7=ONOS7_port)
+
+ mastership_check = main.TRUE
+ for i in range (1,29):
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ try:
+ main.log.info(str(response))
+ except:
+ main.log.info(repr(response))
+ if re.search("tcp:"+ONOS1_ip,response)\
+ and re.search("tcp:"+ONOS2_ip,response)\
+ and re.search("tcp:"+ONOS3_ip,response)\
+ and re.search("tcp:"+ONOS4_ip,response)\
+ and re.search("tcp:"+ONOS5_ip,response)\
+ and re.search("tcp:"+ONOS6_ip,response)\
+ and re.search("tcp:"+ONOS7_ip,response):
+ mastership_check = mastership_check and main.TRUE
+ else:
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Switch mastership assigned correctly")
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership assigned correctly",
+ onfail="Switches not assigned correctly to controllers")
+
+ #TODO: If assign roles is working reliably then manually
+ # assign mastership to the controller we want
+
+
+ def CASE3(self,main) :
+ """
+ Assign intents
+
+ """
+ import time
+ import json
+ import re
+ main.log.report("Adding host intents")
+ main.case("Adding host Intents")
+
+ main.step("Discovering Hosts( Via pingall for now)")
+ #FIXME: Once we have a host discovery mechanism, use that instead
+
+ #REACTIVE FWD test
+ ping_result = main.FALSE
+ time1 = time.time()
+ ping_result = main.Mininet1.pingall()
+ time2 = time.time()
+ main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
+
+ #uninstall onos-app-fwd
+ main.log.info("Uninstall reactive forwarding app")
+ main.ONOScli1.feature_uninstall("onos-app-fwd")
+ main.ONOScli2.feature_uninstall("onos-app-fwd")
+ main.ONOScli3.feature_uninstall("onos-app-fwd")
+ main.ONOScli4.feature_uninstall("onos-app-fwd")
+ main.ONOScli5.feature_uninstall("onos-app-fwd")
+ main.ONOScli6.feature_uninstall("onos-app-fwd")
+ main.ONOScli7.feature_uninstall("onos-app-fwd")
+
+ main.step("Add host intents")
+ #TODO: move the host numbers to params
+ import json
+ intents_json= json.loads(main.ONOScli1.hosts())
+ intent_add_result = main.FALSE
+ for i in range(8,18):
+ main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+ host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+ host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+ #NOTE: get host can return None
+ #TODO: handle this
+ host1_id = main.ONOScli1.get_host(host1)['id']
+ host2_id = main.ONOScli1.get_host(host2)['id']
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ intent_add_result = intent_add_result and tmp_result
+ #TODO Check if intents all exist in datastore
+ #NOTE: Do we need to print this once the test is working?
+ #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
+ # sort_keys=True, indent=4, separators=(',', ': ') ) )
+
+ def CASE4(self,main) :
+ """
+ Ping across added host intents
+ """
+ description = " Ping across added host intents"
+ main.log.report(description)
+ main.case(description)
+ Ping_Result = main.TRUE
+ for i in range(8,18):
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ Ping_Result = Ping_Result and ping
+ if ping==main.FALSE:
+ main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
+ elif ping==main.TRUE:
+ main.log.info("Ping test passed!")
+ Ping_Result = main.TRUE
+ if Ping_Result==main.FALSE:
+ main.log.report("Intents have not been installed correctly, pings failed.")
+ if Ping_Result==main.TRUE:
+ main.log.report("Intents have been installed correctly and verified by pings")
+ utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
+ onpass="Intents have been installed correctly and pings work",
+ onfail ="Intents have not been installed correctly, pings failed." )
+
+ def CASE5(self,main) :
+ '''
+ Reading state of ONOS
+ '''
+ import time
+ import json
+ from subprocess import Popen, PIPE
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+
+ main.log.report("Setting up and gathering data for current state")
+ main.case("Setting up and gathering data for current state")
+ #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
+ #We can then compare them with eachother and also with past states
+
+ main.step("Get the Mastership of each switch from each controller")
+ global mastership_state
+ ONOS1_mastership = main.ONOScli1.roles()
+ ONOS2_mastership = main.ONOScli2.roles()
+ ONOS3_mastership = main.ONOScli3.roles()
+ ONOS4_mastership = main.ONOScli4.roles()
+ ONOS5_mastership = main.ONOScli5.roles()
+ ONOS6_mastership = main.ONOScli6.roles()
+ ONOS7_mastership = main.ONOScli7.roles()
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ if "Error" in ONOS1_mastership or not ONOS1_mastership\
+ or "Error" in ONOS2_mastership or not ONOS2_mastership\
+ or "Error" in ONOS3_mastership or not ONOS3_mastership\
+ or "Error" in ONOS4_mastership or not ONOS4_mastership\
+ or "Error" in ONOS5_mastership or not ONOS5_mastership\
+ or "Error" in ONOS6_mastership or not ONOS6_mastership\
+ or "Error" in ONOS7_mastership or not ONOS7_mastership:
+ main.log.report("Error in getting ONOS roles")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+ main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+ main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+ main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+ main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+ main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+ consistent_mastership = main.FALSE
+ elif ONOS1_mastership == ONOS2_mastership\
+ and ONOS1_mastership == ONOS3_mastership\
+ and ONOS1_mastership == ONOS4_mastership\
+ and ONOS1_mastership == ONOS5_mastership\
+ and ONOS1_mastership == ONOS6_mastership\
+ and ONOS1_mastership == ONOS7_mastership:
+ mastership_state = ONOS1_mastership
+ consistent_mastership = main.TRUE
+ main.log.report("Switch roles are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ consistent_mastership = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+ onpass="Switch roles are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of switch roles")
+
+
+ main.step("Get the intents from each controller")
+ global intent_state
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ ONOS2_intents = main.ONOScli2.intents( json_format=True )
+ ONOS3_intents = main.ONOScli3.intents( json_format=True )
+ ONOS4_intents = main.ONOScli4.intents( json_format=True )
+ ONOS5_intents = main.ONOScli5.intents( json_format=True )
+ ONOS6_intents = main.ONOScli6.intents( json_format=True )
+ ONOS7_intents = main.ONOScli7.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents\
+ or "Error" in ONOS2_intents or not ONOS2_intents\
+ or "Error" in ONOS3_intents or not ONOS3_intents\
+ or "Error" in ONOS4_intents or not ONOS4_intents\
+ or "Error" in ONOS5_intents or not ONOS5_intents\
+ or "Error" in ONOS6_intents or not ONOS6_intents\
+ or "Error" in ONOS7_intents or not ONOS7_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+ main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+ main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+ main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+ main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+ main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+ elif ONOS1_intents == ONOS2_intents\
+ and ONOS1_intents == ONOS3_intents\
+ and ONOS1_intents == ONOS4_intents\
+ and ONOS1_intents == ONOS5_intents\
+ and ONOS1_intents == ONOS6_intents\
+ and ONOS1_intents == ONOS7_intents:
+ intent_state = ONOS1_intents
+ intent_check = main.TRUE
+ main.log.report("Intents are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+ onpass="Intents are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of intents")
+
+
+ main.step("Get the flows from each controller")
+ global flow_state
+ ONOS1_flows = main.ONOScli1.flows( json_format=True )
+ ONOS2_flows = main.ONOScli2.flows( json_format=True )
+ ONOS3_flows = main.ONOScli3.flows( json_format=True )
+ ONOS4_flows = main.ONOScli4.flows( json_format=True )
+ ONOS5_flows = main.ONOScli5.flows( json_format=True )
+ ONOS6_flows = main.ONOScli6.flows( json_format=True )
+ ONOS7_flows = main.ONOScli7.flows( json_format=True )
+ flow_check = main.FALSE
+ if "Error" in ONOS1_flows or not ONOS1_flows\
+ or "Error" in ONOS2_flows or not ONOS2_flows\
+ or "Error" in ONOS3_flows or not ONOS3_flows\
+ or "Error" in ONOS4_flows or not ONOS4_flows\
+ or "Error" in ONOS5_flows or not ONOS5_flows\
+ or "Error" in ONOS6_flows or not ONOS6_flows\
+ or "Error" in ONOS7_flows or not ONOS7_flows:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
+ main.log.warn("ONOS2 flows repsponse: "+ ONOS2_flows)
+ main.log.warn("ONOS3 flows repsponse: "+ ONOS3_flows)
+ main.log.warn("ONOS4 flows repsponse: "+ ONOS4_flows)
+ main.log.warn("ONOS5 flows repsponse: "+ ONOS5_flows)
+ main.log.warn("ONOS6 flows repsponse: "+ ONOS6_flows)
+ main.log.warn("ONOS7 flows repsponse: "+ ONOS7_flows)
+ elif len(json.loads(ONOS1_flows)) == len(json.loads(ONOS2_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS3_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS4_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS5_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS6_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS7_flows)):
+ #TODO: Do a better check, maybe compare flows on switches?
+ flow_state = ONOS1_flows
+ flow_check = main.TRUE
+ main.log.report("Flow count is consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 flows: "+ json.dumps(json.loads(ONOS1_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 flows: "+ json.dumps(json.loads(ONOS2_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 flows: "+ json.dumps(json.loads(ONOS3_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 flows: "+ json.dumps(json.loads(ONOS4_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 flows: "+ json.dumps(json.loads(ONOS5_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 flows: "+ json.dumps(json.loads(ONOS6_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 flows: "+ json.dumps(json.loads(ONOS7_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ utilities.assert_equals(expect = main.TRUE,actual=flow_check,
+ onpass="The flow count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different flow counts")
+
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,29):
+ flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+
+ #TODO: Compare switch flow tables with ONOS flow tables
+
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],
+ target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],
+ target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],
+ target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],
+ target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],
+ target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],
+ target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],
+ target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],
+ target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],
+ target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],
+ target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip' + str(count)) in main.params['CTRL']:
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count = count + 1
+ else:
+ break
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+
+ topo_result = devices_results and ports_results and links_results
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+
+ final_assert = main.TRUE
+ final_assert = final_assert and topo_result and flow_check \
+ and intent_check and consistent_mastership
+ utilities.assert_equals(expect=main.TRUE, actual=final_assert,
+ onpass="State check successful",
+ onfail="State check NOT successful")
+
+
+ def CASE6(self,main) :
+ '''
+ The Failure case.
+ '''
+ main.log.report("Restart entire ONOS cluster")
+ main.log.case("Restart entire ONOS cluster")
+ main.ONOSbench.onos_kill(ONOS1_ip)
+ main.ONOSbench.onos_kill(ONOS2_ip)
+ main.ONOSbench.onos_kill(ONOS3_ip)
+ main.ONOSbench.onos_kill(ONOS4_ip)
+ main.ONOSbench.onos_kill(ONOS5_ip)
+ main.ONOSbench.onos_kill(ONOS6_ip)
+ main.ONOSbench.onos_kill(ONOS7_ip)
+
+ main.step("Checking if ONOS is up yet")
+ count = 0
+ onos_isup_result = main.FALSE
+ while onos_isup_result == main.FALSE and count < 10:
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ count = count + 1
+ # TODO: if it becomes an issue, we can retry this step a few times
+
+
+ cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+ cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+ cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+ cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
+ cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
+ cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
+ cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
+ cli_results = cli_result1 and cli_result2 and cli_result3\
+ and cli_result4 and cli_result5 and cli_result6\
+ and cli_result7
+
+ case_results = main.TRUE and onos_isup_result and cli_results
+ utilities.assert_equals(expect=main.TRUE, actual=case_results,
+ onpass="ONOS restart successful",
+ onfail="ONOS restart NOT successful")
+
+
+ def CASE7(self,main) :
+ '''
+ Check state after ONOS failure
+ '''
+ import os
+ import json
+ main.case("Running ONOS Constant State Tests")
+
+ main.step("Check if switch roles are consistent across all nodes")
+ ONOS1_mastership = main.ONOScli1.roles()
+ ONOS2_mastership = main.ONOScli2.roles()
+ ONOS3_mastership = main.ONOScli3.roles()
+ ONOS4_mastership = main.ONOScli4.roles()
+ ONOS5_mastership = main.ONOScli5.roles()
+ ONOS6_mastership = main.ONOScli6.roles()
+ ONOS7_mastership = main.ONOScli7.roles()
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ if "Error" in ONOS1_mastership or not ONOS1_mastership\
+ or "Error" in ONOS2_mastership or not ONOS2_mastership\
+ or "Error" in ONOS3_mastership or not ONOS3_mastership\
+ or "Error" in ONOS4_mastership or not ONOS4_mastership\
+ or "Error" in ONOS5_mastership or not ONOS5_mastership\
+ or "Error" in ONOS6_mastership or not ONOS6_mastership\
+ or "Error" in ONOS7_mastership or not ONOS7_mastership:
+ main.log.error("Error in getting ONOS mastership")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+ main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+ main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+ main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+ main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+ main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+ consistent_mastership = main.FALSE
+ elif ONOS1_mastership == ONOS2_mastership\
+ and ONOS1_mastership == ONOS3_mastership\
+ and ONOS1_mastership == ONOS4_mastership\
+ and ONOS1_mastership == ONOS5_mastership\
+ and ONOS1_mastership == ONOS6_mastership\
+ and ONOS1_mastership == ONOS7_mastership:
+ #mastership_state = ONOS1_mastership
+ consistent_mastership = main.TRUE
+ main.log.report("Switch roles are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ consistent_mastership = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+ onpass="Switch roles are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of switch roles")
+
+
+ description2 = "Compare switch roles from before failure"
+ main.step(description2)
+
+ current_json = json.loads(ONOS1_mastership)
+ old_json = json.loads(mastership_state)
+ mastership_check = main.TRUE
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+
+ current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
+ old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
+ if current == old:
+ mastership_check = mastership_check and main.TRUE
+ else:
+ main.log.warn("Mastership of switch %s changed" % switchDPID)
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Mastership of Switches was not changed")
+ utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
+ onpass="Mastership of Switches was not changed",
+ onfail="Mastership of some switches changed")
+ #NOTE: we expect mastership to change on controller failure
+ mastership_check = mastership_check #and consistent_mastership
+
+
+
+ main.step("Get the intents and compare across all nodes")
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ ONOS2_intents = main.ONOScli2.intents( json_format=True )
+ ONOS3_intents = main.ONOScli3.intents( json_format=True )
+ ONOS4_intents = main.ONOScli4.intents( json_format=True )
+ ONOS5_intents = main.ONOScli5.intents( json_format=True )
+ ONOS6_intents = main.ONOScli6.intents( json_format=True )
+ ONOS7_intents = main.ONOScli7.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents\
+ or "Error" in ONOS2_intents or not ONOS2_intents\
+ or "Error" in ONOS3_intents or not ONOS3_intents\
+ or "Error" in ONOS4_intents or not ONOS4_intents\
+ or "Error" in ONOS5_intents or not ONOS5_intents\
+ or "Error" in ONOS6_intents or not ONOS6_intents\
+ or "Error" in ONOS7_intents or not ONOS7_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+ main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+ main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+ main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+ main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+ main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+ elif ONOS1_intents == ONOS2_intents\
+ and ONOS1_intents == ONOS3_intents\
+ and ONOS1_intents == ONOS4_intents\
+ and ONOS1_intents == ONOS5_intents\
+ and ONOS1_intents == ONOS6_intents\
+ and ONOS1_intents == ONOS7_intents:
+ intent_check = main.TRUE
+ main.log.report("Intents are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+ onpass="Intents are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of intents")
+
+ main.step("Compare current intents with intents before the failure")
+ if intent_state == ONOS1_intents:
+ same_intents = main.TRUE
+ main.log.report("Intents are consistent with before failure")
+ #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
+ else:
+ same_intents = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=same_intents,
+ onpass="Intents are consistent with before failure",
+ onfail="The Intents changed during failure")
+ intent_check = intent_check and same_intents
+
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ Flow_Tables = main.TRUE
+ flows2=[]
+ for i in range(28):
+ main.log.info("Checking flow table on s" + str(i+1))
+ tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ flows2.append(tmp_flows)
+ Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ if Flow_Tables == main.FALSE:
+ main.log.info("Differences in flow table for switch: "+str(i+1))
+ break
+ if Flow_Tables == main.TRUE:
+ main.log.report("No changes were found in the flow tables")
+ utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
+ onpass="No changes were found in the flow tables",
+ onfail="Changes were found in the flow tables")
+
+ main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+ #FIXME: This check is always failing. Investigate cause
+ #NOTE: this may be something to do with file permsissions
+ # or slight change in format
+ main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
+ Loss_In_Pings = main.FALSE
+ #NOTE: checkForLoss returns main.FALSE with 0% packet loss
+ for i in range(8,18):
+ main.log.info("Checking for a loss in pings along flow from s" + str(i))
+ Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if Loss_In_Pings == main.TRUE:
+ main.log.info("Loss in ping detected")
+ elif Loss_In_Pings == main.ERROR:
+ main.log.info("There are multiple mininet process running")
+ elif Loss_In_Pings == main.FALSE:
+ main.log.info("No Loss in the pings")
+ main.log.report("No loss of dataplane connectivity")
+ utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
+ onpass="No Loss of connectivity",
+ onfail="Loss of dataplane connectivity detected")
+
+
+ #TODO:add topology to this or leave as a seperate case?
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = int(result)
+ if result == main.TRUE:
+ main.log.report("Constant State Tests Passed")
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Constant State Tests Passed",
+ onfail="Constant state tests failed")
+
+ def CASE8 (self,main):
+ '''
+ Compare topo
+ '''
+ import sys
+ sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+ import json
+ import time
+
+ description ="Compare ONOS Topology view to Mininet topology"
+ main.case(description)
+ main.log.report(description)
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip' + str(count)) in main.params['CTRL']:
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count = count + 1
+ else:
+ break
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ topo_result = main.FALSE
+ start_time = time.time()
+ elapsed = 0
+ count = 0
+ while topo_result == main.FALSE and elapsed < 120:
+ count = count + 1
+ try:
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+ except:
+ main.log.error("something went wrong in topo comparison")
+ main.log.warn( repr( devices ) )
+ main.log.warn( repr( ports ) )
+ main.log.warn( repr( links ) )
+
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+ topo_result = devices_results and ports_results and links_results
+ elapsed = time.time() - start_time
+ time_threshold = elapsed < 1
+ topo_result = topo_result and time_threshold
+ #TODO make sure this step is non-blocking. IE add a timeout
+ main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ str(elapsed) + " seconds, " + str(count) +" tries" )
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+ if topo_result == main.TRUE:
+ main.log.report("ONOS topology view matches Mininet topology")
+
+
+ def CASE9 (self,main):
+ '''
+ Link s3-s28 down
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+ description = "Turn off a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+
+ main.step("Kill Link between s3 and s28")
+ Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
+ onpass="Link down succesful",
+ onfail="Failed to bring link down")
+ #TODO do some sort of check here
+
+ def CASE10 (self,main):
+ '''
+ Link s3-s28 up
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+ description = "Restore a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Bring link between s3 and s28 back up")
+ Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
+ onpass="Link up succesful",
+ onfail="Failed to bring link up")
+ #TODO do some sort of check here
+
+
+ def CASE11 (self, main) :
+ '''
+ Switch Down
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+
+ switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+ description = "Killing a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ #TODO: Make this switch parameterizable
+ main.step("Kill s28 ")
+ main.log.report("Deleting s28")
+ #FIXME: use new dynamic topo functions
+ main.Mininet1.del_switch("s28")
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+ time.sleep(switch_sleep)
+ #Peek at the deleted switch
+ main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+ #TODO do some sort of check here
+
+ def CASE12 (self, main) :
+ '''
+ Switch Up
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+ #FIXME: use new dynamic topo functions
+ description = "Adding a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Add back s28")
+ main.log.report("Adding back s28")
+ main.Mininet1.add_switch("s28", dpid = '0000000000002800')
+ #TODO: New dpid or same? Ask Thomas?
+ main.Mininet1.add_link('s28', 's3')
+ main.Mininet1.add_link('s28', 's6')
+ main.Mininet1.add_link('s28', 'h28')
+ main.Mininet1.assign_sw_controller(sw="28",count=7,
+ ip1=ONOS1_ip,port1=ONOS1_port,
+ ip2=ONOS2_ip,port2=ONOS2_port,
+ ip3=ONOS3_ip,port3=ONOS3_port,
+ ip4=ONOS4_ip,port4=ONOS4_port,
+ ip5=ONOS5_ip,port5=ONOS5_port,
+ ip6=ONOS6_ip,port6=ONOS6_port,
+ ip7=ONOS7_ip,port7=ONOS7_port)
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
+ time.sleep(switch_sleep)
+ #Peek at the added switch
+ main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+ #TODO do some sort of check here
+
+ def CASE13 (self, main) :
+ '''
+ Clean up
+ '''
+ import os
+ import time
+ description = "Test Cleanup"
+ main.log.report(description)
+ main.case(description)
+ main.step("Killing tcpdumps")
+ main.Mininet2.stop_tcpdump()
+
+ main.step("Copying MN pcap and ONOS log files to test station")
+ testname = main.TEST
+ #NOTE: MN Pcap file is being saved to ~/packet_captures
+ # scp this file as MN and TestON aren't necessarily the same vm
+ #FIXME: scp
+ #####mn files
+ #TODO: Load these from params
+ #NOTE: must end in /
+ log_folder = "/opt/onos/log/"
+ log_files = ["karaf.log", "karaf.log.1"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS2-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS3-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS4-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS5-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS6-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS7-"+f )
+
+ #std*.log's
+ #NOTE: must end in /
+ log_folder = "/opt/onos/var/"
+ log_files = ["stderr.log", "stdout.log"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS2-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS3-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS4-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS5-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS6-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS7-"+f )
+
+
+
+
+ #sleep so scp can finish
+ time.sleep(10)
+ main.step("Packing and rotating pcap archives")
+ os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
+
+
+ #TODO: actually check something here
+ utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+ onpass="Test cleanup successful",
+ onfail="Test cleanup NOT successful")
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.topo b/TestON/tests/HATestClusterRestart/HATestClusterRestart.topo
new file mode 100644
index 0000000..4d4156c
--- /dev/null
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.topo
@@ -0,0 +1,173 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli3>
+
+
+ <ONOScli4>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli4>
+
+
+ <ONOScli5>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli5>
+
+
+ <ONOScli6>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli6>
+
+
+ <ONOScli7>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli7>
+
+ <ONOS1>
+ <host>10.128.30.11</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>9</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.30.12</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.30.13</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>11</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS3>
+
+ <ONOS4>
+ <host>10.128.30.14</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>12</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS4>
+
+ <ONOS5>
+ <host>10.128.30.15</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS5>
+
+ <ONOS6>
+ <host>10.128.30.16</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS6>
+
+ <ONOS7>
+ <host>10.128.30.17</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>15</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS7>
+
+ <Mininet1>
+ <host>10.128.30.9</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>MininetCliDriver</type>
+ <connect_order>16</connect_order>
+ <COMPONENTS>
+ #Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo </arg2>
+ <arg3> </arg3>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet1>
+
+ <Mininet2>
+ <host>10.128.30.9</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>RemoteMininetDriver</type>
+ <connect_order>17</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet2>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestClusterRestart/__init__.py b/TestON/tests/HATestClusterRestart/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HATestClusterRestart/__init__.py
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
new file mode 100644
index 0000000..0a8bd57
--- /dev/null
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -0,0 +1,63 @@
+<PARAMS>
+ <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <ENV>
+ <cellName>HA</cellName>
+ </ENV>
+ <Git>False</Git>
+
+ <CTRL>
+ <ip1>10.128.30.11</ip1>
+ <port1>6633</port1>
+
+ <ip2>10.128.30.12</ip2>
+ <port2>6633</port2>
+
+ <ip3>10.128.30.13</ip3>
+ <port3>6633</port3>
+
+ <ip4>10.128.30.14</ip4>
+ <port4>6633</port4>
+
+ <ip5>10.128.30.15</ip5>
+ <port5>6633</port5>
+
+ <ip6>10.128.30.16</ip6>
+ <port6>6633</port6>
+
+ <ip7>10.128.30.17</ip7>
+ <port7>6633</port7>
+ </CTRL>
+ <TESTONUSER>admin</TESTONUSER>
+ <TESTONIP>10.128.30.10</TESTONIP>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+ <timers>
+ <LinkDiscovery>2</LinkDiscovery>
+ <SwitchDiscovery>2</SwitchDiscovery>
+ </timers>
+ <MNtcpdump>
+ <intf>eth0</intf>
+ <port> </port>
+ <folder>~/packet_captures/</folder>
+ </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
new file mode 100644
index 0000000..2af775d
--- /dev/null
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -0,0 +1,1202 @@
+'''
+Description: This test is to determine if ONOS can handle
+ a minority of it's nodes restarting
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE6: The Failure case.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+'''
+class HATestMinorityRestart:
+
+ def __init__(self) :
+ self.default = ''
+
+ def CASE1(self,main) :
+ '''
+ CASE1 is to compile ONOS and push it to the test machines
+
+ Startup sequence:
+ git pull
+ mvn clean install
+ onos-package
+ cell <name>
+ onos-verify-cell
+ NOTE: temporary - onos-remove-raft-logs
+ onos-install -f
+ onos-wait-for-start
+ '''
+ import time
+ main.log.report("ONOS HA test: Restart minority of ONOS nodes - initialization")
+ main.case("Setting up test environment")
+
+ # load some vairables from the params file
+ PULL_CODE = False
+ if main.params['Git'] == 'True':
+ PULL_CODE = True
+ cell_name = main.params['ENV']['cellName']
+
+ #set global variables
+ global ONOS1_ip
+ global ONOS1_port
+ global ONOS2_ip
+ global ONOS2_port
+ global ONOS3_ip
+ global ONOS3_port
+ global ONOS4_ip
+ global ONOS4_port
+ global ONOS5_ip
+ global ONOS5_port
+ global ONOS6_ip
+ global ONOS6_port
+ global ONOS7_ip
+ global ONOS7_port
+
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS1_port = main.params['CTRL']['port1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS2_port = main.params['CTRL']['port2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS3_port = main.params['CTRL']['port3']
+ ONOS4_ip = main.params['CTRL']['ip4']
+ ONOS4_port = main.params['CTRL']['port4']
+ ONOS5_ip = main.params['CTRL']['ip5']
+ ONOS5_port = main.params['CTRL']['port5']
+ ONOS6_ip = main.params['CTRL']['ip6']
+ ONOS6_port = main.params['CTRL']['port6']
+ ONOS7_ip = main.params['CTRL']['ip7']
+ ONOS7_port = main.params['CTRL']['port7']
+
+
+ main.step("Applying cell variable to environment")
+ cell_result = main.ONOSbench.set_cell(cell_name)
+ verify_result = main.ONOSbench.verify_cell()
+
+ #FIXME:this is short term fix
+ main.log.report("Removing raft logs")
+ main.ONOSbench.onos_remove_raft_logs()
+ main.log.report("Uninstalling ONOS")
+ main.ONOSbench.onos_uninstall(ONOS1_ip)
+ main.ONOSbench.onos_uninstall(ONOS2_ip)
+ main.ONOSbench.onos_uninstall(ONOS3_ip)
+ main.ONOSbench.onos_uninstall(ONOS4_ip)
+ main.ONOSbench.onos_uninstall(ONOS5_ip)
+ main.ONOSbench.onos_uninstall(ONOS6_ip)
+ main.ONOSbench.onos_uninstall(ONOS7_ip)
+
+ clean_install_result = main.TRUE
+ git_pull_result = main.TRUE
+
+ main.step("Compiling the latest version of ONOS")
+ if PULL_CODE:
+ main.step("Git checkout and pull master")
+ main.ONOSbench.git_checkout("master")
+ git_pull_result = main.ONOSbench.git_pull()
+
+ main.step("Using mvn clean & install")
+ clean_install_result = main.TRUE
+ if git_pull_result == main.TRUE:
+ clean_install_result = main.ONOSbench.clean_install()
+ else:
+ main.log.warn("Did not pull new code so skipping mvn "+ \
+ "clean install")
+ main.ONOSbench.get_version(report=True)
+
+ main.step("Creating ONOS package")
+ package_result = main.ONOSbench.onos_package()
+
+ main.step("Installing ONOS package")
+ onos1_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS1_ip)
+ onos2_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS2_ip)
+ onos3_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS3_ip)
+ onos4_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS4_ip)
+ onos5_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS5_ip)
+ onos6_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS6_ip)
+ onos7_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS7_ip)
+ onos_install_result = onos1_install_result and onos2_install_result\
+ and onos3_install_result and onos4_install_result\
+ and onos5_install_result and onos6_install_result\
+ and onos7_install_result
+ '''
+ #FIXME: work around until onos is less fragile
+ main.ONOSbench.handle.sendline("onos-cluster-install")
+ print main.ONOSbench.handle.expect("\$")
+ onos_install_result = main.TRUE
+ '''
+
+
+ main.step("Checking if ONOS is up yet")
+ #TODO: Refactor
+ # check bundle:list?
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ if not onos2_isup:
+ main.log.report("ONOS2 didn't start!")
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ if not onos3_isup:
+ main.log.report("ONOS3 didn't start!")
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ if not onos4_isup:
+ main.log.report("ONOS4 didn't start!")
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ if not onos5_isup:
+ main.log.report("ONOS5 didn't start!")
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ if not onos6_isup:
+ main.log.report("ONOS6 didn't start!")
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ if not onos7_isup:
+ main.log.report("ONOS7 didn't start!")
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ # TODO: if it becomes an issue, we can retry this step a few times
+
+
+ cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+ cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+ cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+ cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
+ cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
+ cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
+ cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
+ cli_results = cli_result1 and cli_result2 and cli_result3 and\
+ cli_result4 and cli_result5 and cli_result6 and cli_result7
+
+ main.step("Start Packet Capture MN")
+ main.Mininet2.start_tcpdump(
+ str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+ intf = main.params['MNtcpdump']['intf'],
+ port = main.params['MNtcpdump']['port'])
+
+
+ case1_result = (clean_install_result and package_result and
+ cell_result and verify_result and onos_install_result and
+ onos_isup_result and cli_results)
+
+ utilities.assert_equals(expect=main.TRUE, actual=case1_result,
+ onpass="Test startup successful",
+ onfail="Test startup NOT successful")
+
+
+ #if case1_result==main.FALSE:
+ # main.cleanup()
+ # main.exit()
+
+ def CASE2(self,main) :
+ '''
+ Assign mastership to controllers
+ '''
+ import time
+ import json
+ import re
+
+
+ '''
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS1_port = main.params['CTRL']['port1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS2_port = main.params['CTRL']['port2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS3_port = main.params['CTRL']['port3']
+ ONOS4_ip = main.params['CTRL']['ip4']
+ ONOS4_port = main.params['CTRL']['port4']
+ ONOS5_ip = main.params['CTRL']['ip5']
+ ONOS5_port = main.params['CTRL']['port5']
+ ONOS6_ip = main.params['CTRL']['ip6']
+ ONOS6_port = main.params['CTRL']['port6']
+ ONOS7_ip = main.params['CTRL']['ip7']
+ ONOS7_port = main.params['CTRL']['port7']
+ '''
+
+
+ main.log.report("Assigning switches to controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign switches to controllers")
+
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+ ip1=ONOS1_ip,port1=ONOS1_port,
+ ip2=ONOS2_ip,port2=ONOS2_port,
+ ip3=ONOS3_ip,port3=ONOS3_port,
+ ip4=ONOS4_ip,port4=ONOS4_port,
+ ip5=ONOS5_ip,port5=ONOS5_port,
+ ip6=ONOS6_ip,port6=ONOS6_port,
+ ip7=ONOS7_ip,port7=ONOS7_port)
+
+ mastership_check = main.TRUE
+ for i in range (1,29):
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ try:
+ main.log.info(str(response))
+ except:
+ main.log.info(repr(response))
+ if re.search("tcp:"+ONOS1_ip,response)\
+ and re.search("tcp:"+ONOS2_ip,response)\
+ and re.search("tcp:"+ONOS3_ip,response)\
+ and re.search("tcp:"+ONOS4_ip,response)\
+ and re.search("tcp:"+ONOS5_ip,response)\
+ and re.search("tcp:"+ONOS6_ip,response)\
+ and re.search("tcp:"+ONOS7_ip,response):
+ mastership_check = mastership_check and main.TRUE
+ else:
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Switch mastership assigned correctly")
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership assigned correctly",
+ onfail="Switches not assigned correctly to controllers")
+
+ #TODO: If assign roles is working reliably then manually
+ # assign mastership to the controller we want
+
+
+ def CASE3(self,main) :
+ """
+ Assign intents
+
+ """
+ import time
+ import json
+ import re
+ main.log.report("Adding host intents")
+ main.case("Adding host Intents")
+
+ main.step("Discovering Hosts( Via pingall for now)")
+ #FIXME: Once we have a host discovery mechanism, use that instead
+
+ #REACTIVE FWD test
+ ping_result = main.FALSE
+ time1 = time.time()
+ ping_result = main.Mininet1.pingall()
+ time2 = time.time()
+ main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
+
+ #uninstall onos-app-fwd
+ main.log.info("Uninstall reactive forwarding app")
+ main.ONOScli1.feature_uninstall("onos-app-fwd")
+ main.ONOScli2.feature_uninstall("onos-app-fwd")
+ main.ONOScli3.feature_uninstall("onos-app-fwd")
+ main.ONOScli4.feature_uninstall("onos-app-fwd")
+ main.ONOScli5.feature_uninstall("onos-app-fwd")
+ main.ONOScli6.feature_uninstall("onos-app-fwd")
+ main.ONOScli7.feature_uninstall("onos-app-fwd")
+
+ main.step("Add host intents")
+ #TODO: move the host numbers to params
+ import json
+ intents_json= json.loads(main.ONOScli1.hosts())
+ intent_add_result = main.FALSE
+ for i in range(8,18):
+ main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+ host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+ host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+ #NOTE: get host can return None
+ #TODO: handle this
+ host1_id = main.ONOScli1.get_host(host1)['id']
+ host2_id = main.ONOScli1.get_host(host2)['id']
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ intent_add_result = intent_add_result and tmp_result
+ #TODO Check if intents all exist in datastore
+ #NOTE: Do we need to print this once the test is working?
+ #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
+ # sort_keys=True, indent=4, separators=(',', ': ') ) )
+
+ def CASE4(self,main) :
+ """
+ Ping across added host intents
+ """
+ description = " Ping across added host intents"
+ main.log.report(description)
+ main.case(description)
+ Ping_Result = main.TRUE
+ for i in range(8,18):
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ Ping_Result = Ping_Result and ping
+ if ping==main.FALSE:
+ main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
+ elif ping==main.TRUE:
+ main.log.info("Ping test passed!")
+ Ping_Result = main.TRUE
+ if Ping_Result==main.FALSE:
+ main.log.report("Intents have not been installed correctly, pings failed.")
+ if Ping_Result==main.TRUE:
+ main.log.report("Intents have been installed correctly and verified by pings")
+ utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
+ onpass="Intents have been installed correctly and pings work",
+ onfail ="Intents have not been installed correctly, pings failed." )
+
+ def CASE5(self,main) :
+ '''
+ Reading state of ONOS
+ '''
+ import time
+ import json
+ from subprocess import Popen, PIPE
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+
+ main.log.report("Setting up and gathering data for current state")
+ main.case("Setting up and gathering data for current state")
+ #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
+ #We can then compare them with eachother and also with past states
+
+ main.step("Get the Mastership of each switch from each controller")
+ global mastership_state
+ ONOS1_mastership = main.ONOScli1.roles()
+ ONOS2_mastership = main.ONOScli2.roles()
+ ONOS3_mastership = main.ONOScli3.roles()
+ ONOS4_mastership = main.ONOScli4.roles()
+ ONOS5_mastership = main.ONOScli5.roles()
+ ONOS6_mastership = main.ONOScli6.roles()
+ ONOS7_mastership = main.ONOScli7.roles()
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ if "Error" in ONOS1_mastership or not ONOS1_mastership\
+ or "Error" in ONOS2_mastership or not ONOS2_mastership\
+ or "Error" in ONOS3_mastership or not ONOS3_mastership\
+ or "Error" in ONOS4_mastership or not ONOS4_mastership\
+ or "Error" in ONOS5_mastership or not ONOS5_mastership\
+ or "Error" in ONOS6_mastership or not ONOS6_mastership\
+ or "Error" in ONOS7_mastership or not ONOS7_mastership:
+ main.log.report("Error in getting ONOS roles")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+ main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+ main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+ main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+ main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+ main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+ consistent_mastership = main.FALSE
+ elif ONOS1_mastership == ONOS2_mastership\
+ and ONOS1_mastership == ONOS3_mastership\
+ and ONOS1_mastership == ONOS4_mastership\
+ and ONOS1_mastership == ONOS5_mastership\
+ and ONOS1_mastership == ONOS6_mastership\
+ and ONOS1_mastership == ONOS7_mastership:
+ mastership_state = ONOS1_mastership
+ consistent_mastership = main.TRUE
+ main.log.report("Switch roles are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ consistent_mastership = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+ onpass="Switch roles are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of switch roles")
+
+
+ main.step("Get the intents from each controller")
+ global intent_state
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ ONOS2_intents = main.ONOScli2.intents( json_format=True )
+ ONOS3_intents = main.ONOScli3.intents( json_format=True )
+ ONOS4_intents = main.ONOScli4.intents( json_format=True )
+ ONOS5_intents = main.ONOScli5.intents( json_format=True )
+ ONOS6_intents = main.ONOScli6.intents( json_format=True )
+ ONOS7_intents = main.ONOScli7.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents\
+ or "Error" in ONOS2_intents or not ONOS2_intents\
+ or "Error" in ONOS3_intents or not ONOS3_intents\
+ or "Error" in ONOS4_intents or not ONOS4_intents\
+ or "Error" in ONOS5_intents or not ONOS5_intents\
+ or "Error" in ONOS6_intents or not ONOS6_intents\
+ or "Error" in ONOS7_intents or not ONOS7_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+ main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+ main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+ main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+ main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+ main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+ elif ONOS1_intents == ONOS2_intents\
+ and ONOS1_intents == ONOS3_intents\
+ and ONOS1_intents == ONOS4_intents\
+ and ONOS1_intents == ONOS5_intents\
+ and ONOS1_intents == ONOS6_intents\
+ and ONOS1_intents == ONOS7_intents:
+ intent_state = ONOS1_intents
+ intent_check = main.TRUE
+ main.log.report("Intents are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+ onpass="Intents are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of intents")
+
+
+ main.step("Get the flows from each controller")
+ global flow_state
+ ONOS1_flows = main.ONOScli1.flows( json_format=True )
+ ONOS2_flows = main.ONOScli2.flows( json_format=True )
+ ONOS3_flows = main.ONOScli3.flows( json_format=True )
+ ONOS4_flows = main.ONOScli4.flows( json_format=True )
+ ONOS5_flows = main.ONOScli5.flows( json_format=True )
+ ONOS6_flows = main.ONOScli6.flows( json_format=True )
+ ONOS7_flows = main.ONOScli7.flows( json_format=True )
+ flow_check = main.FALSE
+ if "Error" in ONOS1_flows or not ONOS1_flows\
+ or "Error" in ONOS2_flows or not ONOS2_flows\
+ or "Error" in ONOS3_flows or not ONOS3_flows\
+ or "Error" in ONOS4_flows or not ONOS4_flows\
+ or "Error" in ONOS5_flows or not ONOS5_flows\
+ or "Error" in ONOS6_flows or not ONOS6_flows\
+ or "Error" in ONOS7_flows or not ONOS7_flows:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
+ main.log.warn("ONOS2 flows repsponse: "+ ONOS2_flows)
+ main.log.warn("ONOS3 flows repsponse: "+ ONOS3_flows)
+ main.log.warn("ONOS4 flows repsponse: "+ ONOS4_flows)
+ main.log.warn("ONOS5 flows repsponse: "+ ONOS5_flows)
+ main.log.warn("ONOS6 flows repsponse: "+ ONOS6_flows)
+ main.log.warn("ONOS7 flows repsponse: "+ ONOS7_flows)
+ elif len(json.loads(ONOS1_flows)) == len(json.loads(ONOS2_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS3_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS4_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS5_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS6_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS7_flows)):
+ #TODO: Do a better check, maybe compare flows on switches?
+ flow_state = ONOS1_flows
+ flow_check = main.TRUE
+ main.log.report("Flow count is consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 flows: "+ json.dumps(json.loads(ONOS1_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 flows: "+ json.dumps(json.loads(ONOS2_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 flows: "+ json.dumps(json.loads(ONOS3_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 flows: "+ json.dumps(json.loads(ONOS4_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 flows: "+ json.dumps(json.loads(ONOS5_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 flows: "+ json.dumps(json.loads(ONOS6_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 flows: "+ json.dumps(json.loads(ONOS7_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ utilities.assert_equals(expect = main.TRUE,actual=flow_check,
+ onpass="The flow count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different flow counts")
+
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,29):
+ flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+
+ #TODO: Compare switch flow tables with ONOS flow tables
+
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],
+ target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],
+ target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],
+ target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],
+ target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],
+ target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],
+ target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],
+ target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],
+ target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],
+ target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],
+ target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip' + str(count)) in main.params['CTRL']:
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count = count + 1
+ else:
+ break
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+
+ topo_result = devices_results and ports_results and links_results
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+
+ final_assert = main.TRUE
+ final_assert = final_assert and topo_result and flow_check \
+ and intent_check and consistent_mastership
+ utilities.assert_equals(expect=main.TRUE, actual=final_assert,
+ onpass="State check successful",
+ onfail="State check NOT successful")
+
+
+ def CASE6(self,main) :
+ '''
+ The Failure case.
+ '''
+ main.log.report("Killing 3 ONOS nodes")
+ main.log.case("Restart minority of ONOS nodes")
+ #TODO: Randomize these nodes
+ main.ONOSbench.onos_kill(ONOS1_ip)
+ main.ONOSbench.onos_kill(ONOS2_ip)
+ main.ONOSbench.onos_kill(ONOS3_ip)
+
+ main.step("Checking if ONOS is up yet")
+ count = 0
+ onos_isup_result = main.FALSE
+ while onos_isup_result == main.FALSE and count < 10:
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup
+ count = count + 1
+ # TODO: if it becomes an issue, we can retry this step a few times
+
+
+ cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+ cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+ cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+ cli_results = cli_result1 and cli_result2 and cli_result3
+
+ case_results = main.TRUE and onos_isup_result and cli_results
+ utilities.assert_equals(expect=main.TRUE, actual=case_results,
+ onpass="ONOS restart successful",
+ onfail="ONOS restart NOT successful")
+
+
+ def CASE7(self,main) :
+ '''
+ Check state after ONOS failure
+ '''
+ import os
+ import json
+ main.case("Running ONOS Constant State Tests")
+
+ main.step("Check if switch roles are consistent across all nodes")
+ ONOS1_mastership = main.ONOScli1.roles()
+ ONOS2_mastership = main.ONOScli2.roles()
+ ONOS3_mastership = main.ONOScli3.roles()
+ ONOS4_mastership = main.ONOScli4.roles()
+ ONOS5_mastership = main.ONOScli5.roles()
+ ONOS6_mastership = main.ONOScli6.roles()
+ ONOS7_mastership = main.ONOScli7.roles()
+ print type(ONOS1_mastership)
+ print ONOS1_mastership
+ print type(ONOS2_mastership)
+ print ONOS2_mastership
+ print type(ONOS3_mastership)
+ print ONOS3_mastership
+ print type(ONOS4_mastership)
+ print ONOS4_mastership
+ print type(ONOS5_mastership)
+ print ONOS5_mastership
+ print type(ONOS6_mastership)
+ print ONOS6_mastership
+ print type(ONOS7_mastership)
+ print ONOS7_mastership
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ if "Error" in ONOS1_mastership or not ONOS1_mastership\
+ or "Error" in ONOS2_mastership or not ONOS2_mastership\
+ or "Error" in ONOS3_mastership or not ONOS3_mastership\
+ or "Error" in ONOS4_mastership or not ONOS4_mastership\
+ or "Error" in ONOS5_mastership or not ONOS5_mastership\
+ or "Error" in ONOS6_mastership or not ONOS6_mastership\
+ or "Error" in ONOS7_mastership or not ONOS7_mastership:
+ main.log.error("Error in getting ONOS mastership")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+ main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+ main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+ main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+ main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+ main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+ consistent_mastership = main.FALSE
+ elif ONOS1_mastership == ONOS2_mastership\
+ and ONOS1_mastership == ONOS3_mastership\
+ and ONOS1_mastership == ONOS4_mastership\
+ and ONOS1_mastership == ONOS5_mastership\
+ and ONOS1_mastership == ONOS6_mastership\
+ and ONOS1_mastership == ONOS7_mastership:
+ #mastership_state = ONOS1_mastership
+ consistent_mastership = main.TRUE
+ main.log.report("Switch roles are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ consistent_mastership = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+ onpass="Switch roles are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of switch roles")
+
+
+ description2 = "Compare switch roles from before failure"
+ main.step(description2)
+
+ current_json = json.loads(ONOS1_mastership)
+ old_json = json.loads(mastership_state)
+ mastership_check = main.TRUE
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+
+ current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
+ old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
+ if current == old:
+ mastership_check = mastership_check and main.TRUE
+ else:
+ main.log.warn("Mastership of switch %s changed" % switchDPID)
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Mastership of Switches was not changed")
+ utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
+ onpass="Mastership of Switches was not changed",
+ onfail="Mastership of some switches changed")
+ #NOTE: we expect mastership to change on controller failure
+ mastership_check = mastership_check #and consistent_mastership
+
+
+
+ main.step("Get the intents and compare across all nodes")
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ ONOS2_intents = main.ONOScli2.intents( json_format=True )
+ ONOS3_intents = main.ONOScli3.intents( json_format=True )
+ ONOS4_intents = main.ONOScli4.intents( json_format=True )
+ ONOS5_intents = main.ONOScli5.intents( json_format=True )
+ ONOS6_intents = main.ONOScli6.intents( json_format=True )
+ ONOS7_intents = main.ONOScli7.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents\
+ or "Error" in ONOS2_intents or not ONOS2_intents\
+ or "Error" in ONOS3_intents or not ONOS3_intents\
+ or "Error" in ONOS4_intents or not ONOS4_intents\
+ or "Error" in ONOS5_intents or not ONOS5_intents\
+ or "Error" in ONOS6_intents or not ONOS6_intents\
+ or "Error" in ONOS7_intents or not ONOS7_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+ main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+ main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+ main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+ main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+ main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+ elif ONOS1_intents == ONOS2_intents\
+ and ONOS1_intents == ONOS3_intents\
+ and ONOS1_intents == ONOS4_intents\
+ and ONOS1_intents == ONOS5_intents\
+ and ONOS1_intents == ONOS6_intents\
+ and ONOS1_intents == ONOS7_intents:
+ intent_check = main.TRUE
+ main.log.report("Intents are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+ onpass="Intents are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of intents")
+
+ main.step("Compare current intents with intents before the failure")
+ if intent_state == ONOS1_intents:
+ same_intents = main.TRUE
+ main.log.report("Intents are consistent with before failure")
+ #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
+ else:
+ same_intents = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=same_intents,
+ onpass="Intents are consistent with before failure",
+ onfail="The Intents changed during failure")
+ intent_check = intent_check and same_intents
+
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ Flow_Tables = main.TRUE
+ flows2=[]
+ for i in range(28):
+ main.log.info("Checking flow table on s" + str(i+1))
+ tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ flows2.append(tmp_flows)
+ Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ if Flow_Tables == main.FALSE:
+ main.log.info("Differences in flow table for switch: "+str(i+1))
+ break
+ if Flow_Tables == main.TRUE:
+ main.log.report("No changes were found in the flow tables")
+ utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
+ onpass="No changes were found in the flow tables",
+ onfail="Changes were found in the flow tables")
+
+ main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+ #FIXME: This check is always failing. Investigate cause
+ #NOTE: this may be something to do with file permsissions
+ # or slight change in format
+ main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
+ Loss_In_Pings = main.FALSE
+ #NOTE: checkForLoss returns main.FALSE with 0% packet loss
+ for i in range(8,18):
+ main.log.info("Checking for a loss in pings along flow from s" + str(i))
+ Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if Loss_In_Pings == main.TRUE:
+ main.log.info("Loss in ping detected")
+ elif Loss_In_Pings == main.ERROR:
+ main.log.info("There are multiple mininet process running")
+ elif Loss_In_Pings == main.FALSE:
+ main.log.info("No Loss in the pings")
+ main.log.report("No loss of dataplane connectivity")
+ utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
+ onpass="No Loss of connectivity",
+ onfail="Loss of dataplane connectivity detected")
+
+
+ #TODO:add topology to this or leave as a seperate case?
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = int(result)
+ if result == main.TRUE:
+ main.log.report("Constant State Tests Passed")
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Constant State Tests Passed",
+ onfail="Constant state tests failed")
+
+ def CASE8 (self,main):
+ '''
+ Compare topo
+ '''
+ import sys
+ sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+ import json
+ import time
+
+ description ="Compare ONOS Topology view to Mininet topology"
+ main.case(description)
+ main.log.report(description)
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip' + str(count)) in main.params['CTRL']:
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count = count + 1
+ else:
+ break
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ topo_result = main.FALSE
+ start_time = time.time()
+ elapsed = 0
+ count = 0
+ while topo_result == main.FALSE and elapsed < 120:
+ count = count + 1
+ try:
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+ except:
+ main.log.error("something went wrong in topo comparison")
+ main.log.warn( repr( devices ) )
+ main.log.warn( repr( ports ) )
+ main.log.warn( repr( links ) )
+
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+ topo_result = devices_results and ports_results and links_results
+ elapsed = time.time() - start_time
+ time_threshold = elapsed < 1
+ topo_result = topo_result and time_threshold
+ #TODO make sure this step is non-blocking. IE add a timeout
+ main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ str(elapsed) + " seconds, " + str(count) +" tries" )
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+ if topo_result == main.TRUE:
+ main.log.report("ONOS topology view matches Mininet topology")
+
+
+ def CASE9 (self,main):
+ '''
+ Link s3-s28 down
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+ description = "Turn off a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+
+ main.step("Kill Link between s3 and s28")
+ Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
+ onpass="Link down succesful",
+ onfail="Failed to bring link down")
+ #TODO do some sort of check here
+
+ def CASE10 (self,main):
+ '''
+ Link s3-s28 up
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+ description = "Restore a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Bring link between s3 and s28 back up")
+ Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
+ onpass="Link up succesful",
+ onfail="Failed to bring link up")
+ #TODO do some sort of check here
+
+
+ def CASE11 (self, main) :
+ '''
+ Switch Down
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+
+ switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+ description = "Killing a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ #TODO: Make this switch parameterizable
+ main.step("Kill s28 ")
+ main.log.report("Deleting s28")
+ #FIXME: use new dynamic topo functions
+ main.Mininet1.del_switch("s28")
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+ time.sleep(switch_sleep)
+ #Peek at the deleted switch
+ main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+ #TODO do some sort of check here
+
+ def CASE12 (self, main) :
+ '''
+ Switch Up
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+ #FIXME: use new dynamic topo functions
+ description = "Adding a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Add back s28")
+ main.log.report("Adding back s28")
+ main.Mininet1.add_switch("s28", dpid = '0000000000002800')
+ #TODO: New dpid or same? Ask Thomas?
+ main.Mininet1.add_link('s28', 's3')
+ main.Mininet1.add_link('s28', 's6')
+ main.Mininet1.add_link('s28', 'h28')
+ main.Mininet1.assign_sw_controller(sw="28",count=7,
+ ip1=ONOS1_ip,port1=ONOS1_port,
+ ip2=ONOS2_ip,port2=ONOS2_port,
+ ip3=ONOS3_ip,port3=ONOS3_port,
+ ip4=ONOS4_ip,port4=ONOS4_port,
+ ip5=ONOS5_ip,port5=ONOS5_port,
+ ip6=ONOS6_ip,port6=ONOS6_port,
+ ip7=ONOS7_ip,port7=ONOS7_port)
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
+ time.sleep(switch_sleep)
+ #Peek at the added switch
+ main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+ #TODO do some sort of check here
+
+ def CASE13 (self, main) :
+ '''
+ Clean up
+ '''
+ import os
+ import time
+ description = "Test Cleanup"
+ main.log.report(description)
+ main.case(description)
+ main.step("Killing tcpdumps")
+ main.Mininet2.stop_tcpdump()
+
+ main.step("Copying MN pcap and ONOS log files to test station")
+ testname = main.TEST
+ #NOTE: MN Pcap file is being saved to ~/packet_captures
+ # scp this file as MN and TestON aren't necessarily the same vm
+ #FIXME: scp
+ #####mn files
+ #TODO: Load these from params
+ #NOTE: must end in /
+ log_folder = "/opt/onos/log/"
+ log_files = ["karaf.log", "karaf.log.1"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS2-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS3-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS4-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS5-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS6-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS7-"+f )
+
+ #std*.log's
+ #NOTE: must end in /
+ log_folder = "/opt/onos/var/"
+ log_files = ["stderr.log", "stdout.log"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS2-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS3-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS4-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS5-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS6-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS7-"+f )
+
+
+
+
+ #sleep so scp can finish
+ time.sleep(10)
+ main.step("Packing and rotating pcap archives")
+ os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
+
+
+ #TODO: actually check something here
+ utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+ onpass="Test cleanup successful",
+ onfail="Test cleanup NOT successful")
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.topo b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.topo
new file mode 100644
index 0000000..4d4156c
--- /dev/null
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.topo
@@ -0,0 +1,173 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli3>
+
+
+ <ONOScli4>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli4>
+
+
+ <ONOScli5>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli5>
+
+
+ <ONOScli6>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli6>
+
+
+ <ONOScli7>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli7>
+
+ <ONOS1>
+ <host>10.128.30.11</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>9</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.30.12</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.30.13</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>11</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS3>
+
+ <ONOS4>
+ <host>10.128.30.14</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>12</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS4>
+
+ <ONOS5>
+ <host>10.128.30.15</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS5>
+
+ <ONOS6>
+ <host>10.128.30.16</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS6>
+
+ <ONOS7>
+ <host>10.128.30.17</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>15</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS7>
+
+ <Mininet1>
+ <host>10.128.30.9</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>MininetCliDriver</type>
+ <connect_order>16</connect_order>
+ <COMPONENTS>
+ #Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo </arg2>
+ <arg3> </arg3>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet1>
+
+ <Mininet2>
+ <host>10.128.30.9</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>RemoteMininetDriver</type>
+ <connect_order>17</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet2>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestMinorityRestart/__init__.py b/TestON/tests/HATestMinorityRestart/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HATestMinorityRestart/__init__.py
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
new file mode 100644
index 0000000..e1e75f9
--- /dev/null
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -0,0 +1,63 @@
+<PARAMS>
+ <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <ENV>
+ <cellName>HA</cellName>
+ </ENV>
+ <Git>True</Git>
+
+ <CTRL>
+ <ip1>10.128.30.11</ip1>
+ <port1>6633</port1>
+
+ <ip2>10.128.30.12</ip2>
+ <port2>6633</port2>
+
+ <ip3>10.128.30.13</ip3>
+ <port3>6633</port3>
+
+ <ip4>10.128.30.14</ip4>
+ <port4>6633</port4>
+
+ <ip5>10.128.30.15</ip5>
+ <port5>6633</port5>
+
+ <ip6>10.128.30.16</ip6>
+ <port6>6633</port6>
+
+ <ip7>10.128.30.17</ip7>
+ <port7>6633</port7>
+ </CTRL>
+ <TESTONUSER>admin</TESTONUSER>
+ <TESTONIP>10.128.30.10</TESTONIP>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+ <timers>
+ <LinkDiscovery>2</LinkDiscovery>
+ <SwitchDiscovery>2</SwitchDiscovery>
+ </timers>
+ <MNtcpdump>
+ <intf>eth0</intf>
+ <port> </port>
+ <folder>~/packet_captures/</folder>
+ </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
new file mode 100644
index 0000000..614d514
--- /dev/null
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -0,0 +1,1171 @@
+'''
+Description: This test is to determine if the HA test setup is
+ working correctly. There are no failures so this test should
+ have a 100% pass rate
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE6: The Failure case. Since this is the Sanity test, we do nothing.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+'''
+class HATestSanity:
+
+ def __init__(self) :
+ self.default = ''
+
+ def CASE1(self,main) :
+ '''
+ CASE1 is to compile ONOS and push it to the test machines
+
+ Startup sequence:
+ git pull
+ mvn clean install
+ onos-package
+ cell <name>
+ onos-verify-cell
+ NOTE: temporary - onos-remove-raft-logs
+ onos-install -f
+ onos-wait-for-start
+ '''
+ import time
+ main.log.report("ONOS HA Sanity test - initialization")
+ main.case("Setting up test environment")
+ #TODO: save all the timers and output them for plotting
+
+ # load some vairables from the params file
+ PULL_CODE = False
+ if main.params['Git'] == 'True':
+ PULL_CODE = True
+ cell_name = main.params['ENV']['cellName']
+
+ #set global variables
+ global ONOS1_ip
+ global ONOS1_port
+ global ONOS2_ip
+ global ONOS2_port
+ global ONOS3_ip
+ global ONOS3_port
+ global ONOS4_ip
+ global ONOS4_port
+ global ONOS5_ip
+ global ONOS5_port
+ global ONOS6_ip
+ global ONOS6_port
+ global ONOS7_ip
+ global ONOS7_port
+
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS1_port = main.params['CTRL']['port1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS2_port = main.params['CTRL']['port2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS3_port = main.params['CTRL']['port3']
+ ONOS4_ip = main.params['CTRL']['ip4']
+ ONOS4_port = main.params['CTRL']['port4']
+ ONOS5_ip = main.params['CTRL']['ip5']
+ ONOS5_port = main.params['CTRL']['port5']
+ ONOS6_ip = main.params['CTRL']['ip6']
+ ONOS6_port = main.params['CTRL']['port6']
+ ONOS7_ip = main.params['CTRL']['ip7']
+ ONOS7_port = main.params['CTRL']['port7']
+
+
+ main.step("Applying cell variable to environment")
+ cell_result = main.ONOSbench.set_cell(cell_name)
+ verify_result = main.ONOSbench.verify_cell()
+
+ #FIXME:this is short term fix
+ main.log.report("Removing raft logs")
+ main.ONOSbench.onos_remove_raft_logs()
+ main.log.report("Uninstalling ONOS")
+ main.ONOSbench.onos_uninstall(ONOS1_ip)
+ main.ONOSbench.onos_uninstall(ONOS2_ip)
+ main.ONOSbench.onos_uninstall(ONOS3_ip)
+ main.ONOSbench.onos_uninstall(ONOS4_ip)
+ main.ONOSbench.onos_uninstall(ONOS5_ip)
+ main.ONOSbench.onos_uninstall(ONOS6_ip)
+ main.ONOSbench.onos_uninstall(ONOS7_ip)
+
+ clean_install_result = main.TRUE
+ git_pull_result = main.TRUE
+
+ main.step("Compiling the latest version of ONOS")
+ if PULL_CODE:
+ main.step("Git checkout and pull master")
+ main.ONOSbench.git_checkout("master")
+ git_pull_result = main.ONOSbench.git_pull()
+
+ main.step("Using mvn clean & install")
+ clean_install_result = main.TRUE
+ if git_pull_result == main.TRUE:
+ clean_install_result = main.ONOSbench.clean_install()
+ else:
+ main.log.warn("Did not pull new code so skipping mvn "+ \
+ "clean install")
+ main.ONOSbench.get_version(report=True)
+
+ main.step("Creating ONOS package")
+ package_result = main.ONOSbench.onos_package()
+
+ main.step("Installing ONOS package")
+ onos1_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS1_ip)
+ onos2_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS2_ip)
+ onos3_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS3_ip)
+ onos4_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS4_ip)
+ onos5_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS5_ip)
+ onos6_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS6_ip)
+ onos7_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS7_ip)
+ onos_install_result = onos1_install_result and onos2_install_result\
+ and onos3_install_result and onos4_install_result\
+ and onos5_install_result and onos6_install_result\
+ and onos7_install_result
+ '''
+ #FIXME: work around until onos is less fragile
+ main.ONOSbench.handle.sendline("onos-cluster-install")
+ print main.ONOSbench.handle.expect("\$")
+ onos_install_result = main.TRUE
+ '''
+
+
+ main.step("Checking if ONOS is up yet")
+ #TODO: Refactor
+ # check bundle:list?
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ if not onos2_isup:
+ main.log.report("ONOS2 didn't start!")
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ if not onos3_isup:
+ main.log.report("ONOS3 didn't start!")
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ if not onos4_isup:
+ main.log.report("ONOS4 didn't start!")
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ if not onos5_isup:
+ main.log.report("ONOS5 didn't start!")
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ if not onos6_isup:
+ main.log.report("ONOS6 didn't start!")
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ if not onos7_isup:
+ main.log.report("ONOS7 didn't start!")
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ if onos_isup_result == main.TRUE:
+ break
+ # TODO: if it becomes an issue, we can retry this step a few times
+
+
+ cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+ cli_result2 = main.ONOScli2.start_onos_cli(ONOS2_ip)
+ cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
+ cli_result4 = main.ONOScli4.start_onos_cli(ONOS4_ip)
+ cli_result5 = main.ONOScli5.start_onos_cli(ONOS5_ip)
+ cli_result6 = main.ONOScli6.start_onos_cli(ONOS6_ip)
+ cli_result7 = main.ONOScli7.start_onos_cli(ONOS7_ip)
+ cli_results = cli_result1 and cli_result2 and cli_result3 and\
+ cli_result4 and cli_result5 and cli_result6 and cli_result7
+
+ main.step("Start Packet Capture MN")
+ main.Mininet2.start_tcpdump(
+ str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+ intf = main.params['MNtcpdump']['intf'],
+ port = main.params['MNtcpdump']['port'])
+
+
+ case1_result = (clean_install_result and package_result and
+ cell_result and verify_result and onos_install_result and
+ onos_isup_result and cli_results)
+
+ utilities.assert_equals(expect=main.TRUE, actual=case1_result,
+ onpass="Test startup successful",
+ onfail="Test startup NOT successful")
+
+
+ if case1_result==main.FALSE:
+ main.cleanup()
+ main.exit()
+
+ def CASE2(self,main) :
+ '''
+ Assign mastership to controllers
+ '''
+ import time
+ import json
+ import re
+
+
+ '''
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS1_port = main.params['CTRL']['port1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS2_port = main.params['CTRL']['port2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS3_port = main.params['CTRL']['port3']
+ ONOS4_ip = main.params['CTRL']['ip4']
+ ONOS4_port = main.params['CTRL']['port4']
+ ONOS5_ip = main.params['CTRL']['ip5']
+ ONOS5_port = main.params['CTRL']['port5']
+ ONOS6_ip = main.params['CTRL']['ip6']
+ ONOS6_port = main.params['CTRL']['port6']
+ ONOS7_ip = main.params['CTRL']['ip7']
+ ONOS7_port = main.params['CTRL']['port7']
+ '''
+
+
+ main.log.report("Assigning switches to controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign switches to controllers")
+
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+ ip1=ONOS1_ip,port1=ONOS1_port,
+ ip2=ONOS2_ip,port2=ONOS2_port,
+ ip3=ONOS3_ip,port3=ONOS3_port,
+ ip4=ONOS4_ip,port4=ONOS4_port,
+ ip5=ONOS5_ip,port5=ONOS5_port,
+ ip6=ONOS6_ip,port6=ONOS6_port,
+ ip7=ONOS7_ip,port7=ONOS7_port)
+
+ mastership_check = main.TRUE
+ for i in range (1,29):
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ try:
+ main.log.info(str(response))
+ except:
+ main.log.info(repr(response))
+ if re.search("tcp:"+ONOS1_ip,response)\
+ and re.search("tcp:"+ONOS2_ip,response)\
+ and re.search("tcp:"+ONOS3_ip,response)\
+ and re.search("tcp:"+ONOS4_ip,response)\
+ and re.search("tcp:"+ONOS5_ip,response)\
+ and re.search("tcp:"+ONOS6_ip,response)\
+ and re.search("tcp:"+ONOS7_ip,response):
+ mastership_check = mastership_check and main.TRUE
+ else:
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Switch mastership assigned correctly")
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership assigned correctly",
+ onfail="Switches not assigned correctly to controllers")
+
+ #TODO: If assign roles is working reliably then manually
+ # assign mastership to the controller we want
+
+
+ def CASE3(self,main) :
+ """
+ Assign intents
+
+ """
+ import time
+ import json
+ import re
+ main.log.report("Adding host intents")
+ main.case("Adding host Intents")
+
+ main.step("Discovering Hosts( Via pingall for now)")
+ #FIXME: Once we have a host discovery mechanism, use that instead
+
+ #REACTIVE FWD test
+ ping_result = main.FALSE
+ time1 = time.time()
+ ping_result = main.Mininet1.pingall()
+ time2 = time.time()
+ main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
+
+ #uninstall onos-app-fwd
+ main.log.info("Uninstall reactive forwarding app")
+ main.ONOScli1.feature_uninstall("onos-app-fwd")
+ main.ONOScli2.feature_uninstall("onos-app-fwd")
+ main.ONOScli3.feature_uninstall("onos-app-fwd")
+ main.ONOScli4.feature_uninstall("onos-app-fwd")
+ main.ONOScli5.feature_uninstall("onos-app-fwd")
+ main.ONOScli6.feature_uninstall("onos-app-fwd")
+ main.ONOScli7.feature_uninstall("onos-app-fwd")
+
+ main.step("Add host intents")
+ #TODO: move the host numbers to params
+ import json
+ intents_json= json.loads(main.ONOScli1.hosts())
+ intent_add_result = main.FALSE
+ for i in range(8,18):
+ main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+ host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+ host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+ #NOTE: get host can return None
+ #TODO: handle this
+ host1_id = main.ONOScli1.get_host(host1)['id']
+ host2_id = main.ONOScli1.get_host(host2)['id']
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ intent_add_result = intent_add_result and tmp_result
+ #TODO Check if intents all exist in datastore
+ #NOTE: Do we need to print this once the test is working?
+ #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
+ # sort_keys=True, indent=4, separators=(',', ': ') ) )
+
+ def CASE4(self,main) :
+ """
+ Ping across added host intents
+ """
+ description = " Ping across added host intents"
+ main.log.report(description)
+ main.case(description)
+ Ping_Result = main.TRUE
+ for i in range(8,18):
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ Ping_Result = Ping_Result and ping
+ if ping==main.FALSE:
+ main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
+ elif ping==main.TRUE:
+ main.log.info("Ping test passed!")
+ Ping_Result = main.TRUE
+ if Ping_Result==main.FALSE:
+ main.log.report("Intents have not been installed correctly, pings failed.")
+ if Ping_Result==main.TRUE:
+ main.log.report("Intents have been installed correctly and verified by pings")
+ utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
+ onpass="Intents have been installed correctly and pings work",
+ onfail ="Intents have not been installed correctly, pings failed." )
+
+ def CASE5(self,main) :
+ '''
+ Reading state of ONOS
+ '''
+ import time
+ import json
+ from subprocess import Popen, PIPE
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+
+ main.log.report("Setting up and gathering data for current state")
+ main.case("Setting up and gathering data for current state")
+ #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
+ #We can then compare them with eachother and also with past states
+
+ main.step("Get the Mastership of each switch from each controller")
+ global mastership_state
+ ONOS1_mastership = main.ONOScli1.roles()
+ ONOS2_mastership = main.ONOScli2.roles()
+ ONOS3_mastership = main.ONOScli3.roles()
+ ONOS4_mastership = main.ONOScli4.roles()
+ ONOS5_mastership = main.ONOScli5.roles()
+ ONOS6_mastership = main.ONOScli6.roles()
+ ONOS7_mastership = main.ONOScli7.roles()
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ if "Error" in ONOS1_mastership or not ONOS1_mastership\
+ or "Error" in ONOS2_mastership or not ONOS2_mastership\
+ or "Error" in ONOS3_mastership or not ONOS3_mastership\
+ or "Error" in ONOS4_mastership or not ONOS4_mastership\
+ or "Error" in ONOS5_mastership or not ONOS5_mastership\
+ or "Error" in ONOS6_mastership or not ONOS6_mastership\
+ or "Error" in ONOS7_mastership or not ONOS7_mastership:
+ main.log.report("Error in getting ONOS roles")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+ main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+ main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+ main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+ main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+ main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+ consistent_mastership = main.FALSE
+ elif ONOS1_mastership == ONOS2_mastership\
+ and ONOS1_mastership == ONOS3_mastership\
+ and ONOS1_mastership == ONOS4_mastership\
+ and ONOS1_mastership == ONOS5_mastership\
+ and ONOS1_mastership == ONOS6_mastership\
+ and ONOS1_mastership == ONOS7_mastership:
+ mastership_state = ONOS1_mastership
+ consistent_mastership = main.TRUE
+ main.log.report("Switch roles are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ consistent_mastership = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+ onpass="Switch roles are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of switch roles")
+
+
+ main.step("Get the intents from each controller")
+ global intent_state
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ ONOS2_intents = main.ONOScli2.intents( json_format=True )
+ ONOS3_intents = main.ONOScli3.intents( json_format=True )
+ ONOS4_intents = main.ONOScli4.intents( json_format=True )
+ ONOS5_intents = main.ONOScli5.intents( json_format=True )
+ ONOS6_intents = main.ONOScli6.intents( json_format=True )
+ ONOS7_intents = main.ONOScli7.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents\
+ or "Error" in ONOS2_intents or not ONOS2_intents\
+ or "Error" in ONOS3_intents or not ONOS3_intents\
+ or "Error" in ONOS4_intents or not ONOS4_intents\
+ or "Error" in ONOS5_intents or not ONOS5_intents\
+ or "Error" in ONOS6_intents or not ONOS6_intents\
+ or "Error" in ONOS7_intents or not ONOS7_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+ main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+ main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+ main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+ main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+ main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+ elif ONOS1_intents == ONOS2_intents\
+ and ONOS1_intents == ONOS3_intents\
+ and ONOS1_intents == ONOS4_intents\
+ and ONOS1_intents == ONOS5_intents\
+ and ONOS1_intents == ONOS6_intents\
+ and ONOS1_intents == ONOS7_intents:
+ intent_state = ONOS1_intents
+ intent_check = main.TRUE
+ main.log.report("Intents are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+ onpass="Intents are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of intents")
+
+
+ main.step("Get the flows from each controller")
+ global flow_state
+ ONOS1_flows = main.ONOScli1.flows( json_format=True )
+ ONOS2_flows = main.ONOScli2.flows( json_format=True )
+ ONOS3_flows = main.ONOScli3.flows( json_format=True )
+ ONOS4_flows = main.ONOScli4.flows( json_format=True )
+ ONOS5_flows = main.ONOScli5.flows( json_format=True )
+ ONOS6_flows = main.ONOScli6.flows( json_format=True )
+ ONOS7_flows = main.ONOScli7.flows( json_format=True )
+ flow_check = main.FALSE
+ if "Error" in ONOS1_flows or not ONOS1_flows\
+ or "Error" in ONOS2_flows or not ONOS2_flows\
+ or "Error" in ONOS3_flows or not ONOS3_flows\
+ or "Error" in ONOS4_flows or not ONOS4_flows\
+ or "Error" in ONOS5_flows or not ONOS5_flows\
+ or "Error" in ONOS6_flows or not ONOS6_flows\
+ or "Error" in ONOS7_flows or not ONOS7_flows:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
+ main.log.warn("ONOS2 flows repsponse: "+ ONOS2_flows)
+ main.log.warn("ONOS3 flows repsponse: "+ ONOS3_flows)
+ main.log.warn("ONOS4 flows repsponse: "+ ONOS4_flows)
+ main.log.warn("ONOS5 flows repsponse: "+ ONOS5_flows)
+ main.log.warn("ONOS6 flows repsponse: "+ ONOS6_flows)
+ main.log.warn("ONOS7 flows repsponse: "+ ONOS7_flows)
+ elif len(json.loads(ONOS1_flows)) == len(json.loads(ONOS2_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS3_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS4_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS5_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS6_flows))\
+ and len(json.loads(ONOS1_flows)) == len(json.loads(ONOS7_flows)):
+ #TODO: Do a better check, maybe compare flows on switches?
+ flow_state = ONOS1_flows
+ flow_check = main.TRUE
+ main.log.report("Flow count is consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 flows: "+ json.dumps(json.loads(ONOS1_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 flows: "+ json.dumps(json.loads(ONOS2_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 flows: "+ json.dumps(json.loads(ONOS3_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 flows: "+ json.dumps(json.loads(ONOS4_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 flows: "+ json.dumps(json.loads(ONOS5_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 flows: "+ json.dumps(json.loads(ONOS6_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 flows: "+ json.dumps(json.loads(ONOS7_flows),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ utilities.assert_equals(expect = main.TRUE,actual=flow_check,
+ onpass="The flow count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different flow counts")
+
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,29):
+ flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+
+ #TODO: Compare switch flow tables with ONOS flow tables
+
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],
+ target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],
+ target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],
+ target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],
+ target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],
+ target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],
+ target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],
+ target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],
+ target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],
+ target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],
+ target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip' + str(count)) in main.params['CTRL']:
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count = count + 1
+ else:
+ break
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+
+ topo_result = devices_results and ports_results and links_results
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+
+ final_assert = main.TRUE
+ final_assert = final_assert and topo_result and flow_check \
+ and intent_check and consistent_mastership
+ utilities.assert_equals(expect=main.TRUE, actual=final_assert,
+ onpass="State check successful",
+ onfail="State check NOT successful")
+
+
+ def CASE6(self,main) :
+ '''
+ The Failure case. Since this is the Sanity test, we do nothing.
+ '''
+ import time
+ main.log.report("Wait 60 seconds instead of inducing a failure")
+ time.sleep(60)
+ utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+ onpass="Sleeping 60 seconds",
+ onfail="Something is terribly wrong with my math")
+
+ def CASE7(self,main) :
+ '''
+ Check state after ONOS failure
+ '''
+ import os
+ import json
+ main.case("Running ONOS Constant State Tests")
+
+ main.step("Check if switch roles are consistent across all nodes")
+ ONOS1_mastership = main.ONOScli1.roles()
+ ONOS2_mastership = main.ONOScli2.roles()
+ ONOS3_mastership = main.ONOScli3.roles()
+ ONOS4_mastership = main.ONOScli4.roles()
+ ONOS5_mastership = main.ONOScli5.roles()
+ ONOS6_mastership = main.ONOScli6.roles()
+ ONOS7_mastership = main.ONOScli7.roles()
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ if "Error" in ONOS1_mastership or not ONOS1_mastership\
+ or "Error" in ONOS2_mastership or not ONOS2_mastership\
+ or "Error" in ONOS3_mastership or not ONOS3_mastership\
+ or "Error" in ONOS4_mastership or not ONOS4_mastership\
+ or "Error" in ONOS5_mastership or not ONOS5_mastership\
+ or "Error" in ONOS6_mastership or not ONOS6_mastership\
+ or "Error" in ONOS7_mastership or not ONOS7_mastership:
+ main.log.error("Error in getting ONOS mastership")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ main.log.warn("ONOS2 mastership response: " + repr(ONOS2_mastership))
+ main.log.warn("ONOS3 mastership response: " + repr(ONOS3_mastership))
+ main.log.warn("ONOS4 mastership response: " + repr(ONOS4_mastership))
+ main.log.warn("ONOS5 mastership response: " + repr(ONOS5_mastership))
+ main.log.warn("ONOS6 mastership response: " + repr(ONOS6_mastership))
+ main.log.warn("ONOS7 mastership response: " + repr(ONOS7_mastership))
+ consistent_mastership = main.FALSE
+ elif ONOS1_mastership == ONOS2_mastership\
+ and ONOS1_mastership == ONOS3_mastership\
+ and ONOS1_mastership == ONOS4_mastership\
+ and ONOS1_mastership == ONOS5_mastership\
+ and ONOS1_mastership == ONOS6_mastership\
+ and ONOS1_mastership == ONOS7_mastership:
+ #mastership_state = ONOS1_mastership
+ consistent_mastership = main.TRUE
+ main.log.report("Switch roles are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 roles: ", json.dumps(json.loads(ONOS1_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 roles: ", json.dumps(json.loads(ONOS2_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 roles: ", json.dumps(json.loads(ONOS3_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 roles: ", json.dumps(json.loads(ONOS4_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 roles: ", json.dumps(json.loads(ONOS5_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 roles: ", json.dumps(json.loads(ONOS6_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 roles: ", json.dumps(json.loads(ONOS7_mastership),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ consistent_mastership = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+ onpass="Switch roles are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of switch roles")
+
+
+ description2 = "Compare switch roles from before failure"
+ main.step(description2)
+
+ current_json = json.loads(ONOS1_mastership)
+ old_json = json.loads(mastership_state)
+ mastership_check = main.TRUE
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+
+ current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
+ old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
+ if current == old:
+ mastership_check = mastership_check and main.TRUE
+ else:
+ main.log.warn("Mastership of switch %s changed" % switchDPID)
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Mastership of Switches was not changed")
+ utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
+ onpass="Mastership of Switches was not changed",
+ onfail="Mastership of some switches changed")
+ mastership_check = mastership_check and consistent_mastership
+
+
+
+ main.step("Get the intents and compare across all nodes")
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ ONOS2_intents = main.ONOScli2.intents( json_format=True )
+ ONOS3_intents = main.ONOScli3.intents( json_format=True )
+ ONOS4_intents = main.ONOScli4.intents( json_format=True )
+ ONOS5_intents = main.ONOScli5.intents( json_format=True )
+ ONOS6_intents = main.ONOScli6.intents( json_format=True )
+ ONOS7_intents = main.ONOScli7.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents\
+ or "Error" in ONOS2_intents or not ONOS2_intents\
+ or "Error" in ONOS3_intents or not ONOS3_intents\
+ or "Error" in ONOS4_intents or not ONOS4_intents\
+ or "Error" in ONOS5_intents or not ONOS5_intents\
+ or "Error" in ONOS6_intents or not ONOS6_intents\
+ or "Error" in ONOS7_intents or not ONOS7_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ main.log.warn("ONOS2 intents response: " + repr(ONOS2_intents))
+ main.log.warn("ONOS3 intents response: " + repr(ONOS3_intents))
+ main.log.warn("ONOS4 intents response: " + repr(ONOS4_intents))
+ main.log.warn("ONOS5 intents response: " + repr(ONOS5_intents))
+ main.log.warn("ONOS6 intents response: " + repr(ONOS6_intents))
+ main.log.warn("ONOS7 intents response: " + repr(ONOS7_intents))
+ elif ONOS1_intents == ONOS2_intents\
+ and ONOS1_intents == ONOS3_intents\
+ and ONOS1_intents == ONOS4_intents\
+ and ONOS1_intents == ONOS5_intents\
+ and ONOS1_intents == ONOS6_intents\
+ and ONOS1_intents == ONOS7_intents:
+ intent_check = main.TRUE
+ main.log.report("Intents are consistent across all ONOS nodes")
+ else:
+ main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': ')))
+ utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+ onpass="Intents are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of intents")
+
+ main.step("Compare current intents with intents before the failure")
+ if intent_state == ONOS1_intents:
+ same_intents = main.TRUE
+ main.log.report("Intents are consistent with before failure")
+ #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
+ else:
+ same_intents = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=same_intents,
+ onpass="Intents are consistent with before failure",
+ onfail="The Intents changed during failure")
+ intent_check = intent_check and same_intents
+
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ Flow_Tables = main.TRUE
+ flows2=[]
+ for i in range(28):
+ main.log.info("Checking flow table on s" + str(i+1))
+ tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ flows2.append(tmp_flows)
+ Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ if Flow_Tables == main.FALSE:
+ main.log.info("Differences in flow table for switch: "+str(i+1))
+ break
+ if Flow_Tables == main.TRUE:
+ main.log.report("No changes were found in the flow tables")
+ utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
+ onpass="No changes were found in the flow tables",
+ onfail="Changes were found in the flow tables")
+
+ main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+ #FIXME: This check is always failing. Investigate cause
+ #NOTE: this may be something to do with file permsissions
+ # or slight change in format
+ main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
+ Loss_In_Pings = main.FALSE
+ #NOTE: checkForLoss returns main.FALSE with 0% packet loss
+ for i in range(8,18):
+ main.log.info("Checking for a loss in pings along flow from s" + str(i))
+ Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if Loss_In_Pings == main.TRUE:
+ main.log.info("Loss in ping detected")
+ elif Loss_In_Pings == main.ERROR:
+ main.log.info("There are multiple mininet process running")
+ elif Loss_In_Pings == main.FALSE:
+ main.log.info("No Loss in the pings")
+ main.log.report("No loss of dataplane connectivity")
+ utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
+ onpass="No Loss of connectivity",
+ onfail="Loss of dataplane connectivity detected")
+
+
+ #TODO:add topology to this or leave as a seperate case?
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = int(result)
+ if result == main.TRUE:
+ main.log.report("Constant State Tests Passed")
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Constant State Tests Passed",
+ onfail="Constant state tests failed")
+
+ def CASE8 (self,main):
+ '''
+ Compare topo
+ '''
+ import sys
+ sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+ import json
+ import time
+
+ description ="Compare ONOS Topology view to Mininet topology"
+ main.case(description)
+ main.log.report(description)
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip' + str(count)) in main.params['CTRL']:
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count = count + 1
+ else:
+ break
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ topo_result = main.FALSE
+ start_time = time.time()
+ elapsed = 0
+ count = 0
+ while topo_result == main.FALSE and elapsed < 120:
+ print "cond 1:" + str(topo_result == main.FALSE)
+ print "cond 2:" + str(elapsed < 120)
+ count = count + 1
+ try:
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+ except:
+ main.log.error("something went wrong in topo comparison")
+ main.log.warn( repr( devices ) )
+ main.log.warn( repr( ports ) )
+ main.log.warn( repr( links ) )
+
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+ topo_result = devices_results and ports_results and links_results
+ elapsed = time.time() - start_time
+ time_threshold = elapsed < 1
+ topo_result = topo_result and time_threshold
+ #TODO make sure this step is non-blocking. IE add a timeout
+ main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ str(elapsed) + " seconds, " + str(count) +" tries" )
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+ if topo_result == main.TRUE:
+ main.log.report("ONOS topology view matches Mininet topology")
+
+
+ def CASE9 (self,main):
+ '''
+ Link s3-s28 down
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+ description = "Turn off a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+
+ main.step("Kill Link between s3 and s28")
+ Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
+ onpass="Link down succesful",
+ onfail="Failed to bring link down")
+ #TODO do some sort of check here
+
+ def CASE10 (self,main):
+ '''
+ Link s3-s28 up
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+ description = "Restore a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Bring link between s3 and s28 back up")
+ Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
+ onpass="Link up succesful",
+ onfail="Failed to bring link up")
+ #TODO do some sort of check here
+
+
+ def CASE11 (self, main) :
+ '''
+ Switch Down
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+
+ switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+ description = "Killing a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ #TODO: Make this switch parameterizable
+ main.step("Kill s28 ")
+ main.log.report("Deleting s28")
+ #FIXME: use new dynamic topo functions
+ main.Mininet1.del_switch("s28")
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+ time.sleep(switch_sleep)
+ #Peek at the deleted switch
+ main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+ #TODO do some sort of check here
+
+ def CASE12 (self, main) :
+ '''
+ Switch Up
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+ #FIXME: use new dynamic topo functions
+ description = "Adding a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Add back s28")
+ main.log.report("Adding back s28")
+ main.Mininet1.add_switch("s28", dpid = '0000000000002800')
+ #TODO: New dpid or same? Ask Thomas?
+ main.Mininet1.add_link('s28', 's3')
+ main.Mininet1.add_link('s28', 's6')
+ main.Mininet1.add_link('s28', 'h28')
+ main.Mininet1.assign_sw_controller(sw="28",count=7,
+ ip1=ONOS1_ip,port1=ONOS1_port,
+ ip2=ONOS2_ip,port2=ONOS2_port,
+ ip3=ONOS3_ip,port3=ONOS3_port,
+ ip4=ONOS4_ip,port4=ONOS4_port,
+ ip5=ONOS5_ip,port5=ONOS5_port,
+ ip6=ONOS6_ip,port6=ONOS6_port,
+ ip7=ONOS7_ip,port7=ONOS7_port)
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
+ time.sleep(switch_sleep)
+ #Peek at the added switch
+ main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+ #TODO do some sort of check here
+
+ def CASE13 (self, main) :
+ '''
+ Clean up
+ '''
+ import os
+ import time
+ description = "Test Cleanup"
+ main.log.report(description)
+ main.case(description)
+ main.step("Killing tcpdumps")
+ main.Mininet2.stop_tcpdump()
+
+ main.step("Copying MN pcap and ONOS log files to test station")
+ testname = main.TEST
+ #NOTE: MN Pcap file is being saved to ~/packet_captures
+ # scp this file as MN and TestON aren't necessarily the same vm
+ #FIXME: scp
+ #####mn files
+ #TODO: Load these from params
+ #NOTE: must end in /
+ log_folder = "/opt/onos/log/"
+ log_files = ["karaf.log", "karaf.log.1"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS2-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS3-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS4-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS5-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS6-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS7-"+f )
+
+ #std*.log's
+ #NOTE: must end in /
+ log_folder = "/opt/onos/var/"
+ log_files = ["stderr.log", "stdout.log"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS2-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS3-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS4-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS5-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS6-"+f )
+ main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS7-"+f )
+
+
+
+
+ #sleep so scp can finish
+ time.sleep(10)
+ main.step("Packing and rotating pcap archives")
+ os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
+
+
+ #TODO: actually check something here
+ utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+ onpass="Test cleanup successful",
+ onfail="Test cleanup NOT successful")
diff --git a/TestON/tests/HATestSanity/HATestSanity.topo b/TestON/tests/HATestSanity/HATestSanity.topo
new file mode 100644
index 0000000..4d4156c
--- /dev/null
+++ b/TestON/tests/HATestSanity/HATestSanity.topo
@@ -0,0 +1,173 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli3>
+
+
+ <ONOScli4>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli4>
+
+
+ <ONOScli5>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli5>
+
+
+ <ONOScli6>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli6>
+
+
+ <ONOScli7>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli7>
+
+ <ONOS1>
+ <host>10.128.30.11</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>9</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.30.12</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.30.13</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>11</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS3>
+
+ <ONOS4>
+ <host>10.128.30.14</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>12</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS4>
+
+ <ONOS5>
+ <host>10.128.30.15</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS5>
+
+ <ONOS6>
+ <host>10.128.30.16</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS6>
+
+ <ONOS7>
+ <host>10.128.30.17</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>15</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS7>
+
+ <Mininet1>
+ <host>10.128.30.9</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>MininetCliDriver</type>
+ <connect_order>16</connect_order>
+ <COMPONENTS>
+ #Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo </arg2>
+ <arg3> </arg3>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet1>
+
+ <Mininet2>
+ <host>10.128.30.9</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>RemoteMininetDriver</type>
+ <connect_order>17</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet2>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestSanity/__init__.py b/TestON/tests/HATestSanity/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HATestSanity/__init__.py
diff --git a/TestON/tests/IntentPerfNext/IntentPerfNext.params b/TestON/tests/IntentPerfNext/IntentPerfNext.params
index 7e1de77..87e7998 100644
--- a/TestON/tests/IntentPerfNext/IntentPerfNext.params
+++ b/TestON/tests/IntentPerfNext/IntentPerfNext.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2</testcases>
+ <testcases>1,4</testcases>
<ENV>
<cellName>intent_perf_test</cellName>
@@ -32,7 +32,8 @@
<TEST>
#Number of times to iterate each case
- <numIter>5</numIter>
+ <numIter>3</numIter>
+ <batchIntentSize>50</batchIntentSize>
</TEST>
<JSON>
diff --git a/TestON/tests/IntentPerfNext/IntentPerfNext.py b/TestON/tests/IntentPerfNext/IntentPerfNext.py
index 2fb22dc..dc9fadc 100644
--- a/TestON/tests/IntentPerfNext/IntentPerfNext.py
+++ b/TestON/tests/IntentPerfNext/IntentPerfNext.py
@@ -31,7 +31,8 @@
main.step("Creating cell file")
cell_file_result = main.ONOSbench.create_cell_file(
- BENCH_ip, cell_name, MN1_ip, "onos-core",
+ BENCH_ip, cell_name, MN1_ip,
+ "onos-core,onos-app-metrics,onos-gui",
ONOS1_ip, ONOS2_ip, ONOS3_ip)
main.step("Applying cell file to environment")
@@ -53,6 +54,9 @@
build_result = main.TRUE
main.log.info("Git pull skipped by configuration")
+ main.log.report("Commit information - ")
+ main.ONOSbench.get_version(report=True)
+
main.step("Creating ONOS package")
package_result = main.ONOSbench.onos_package()
@@ -73,11 +77,6 @@
cli2 = main.ONOS2cli.start_onos_cli(ONOS2_ip)
cli3 = main.ONOS3cli.start_onos_cli(ONOS3_ip)
- main.step("Enable metrics feature")
- main.ONOS1cli.feature_install("onos-app-metrics")
- main.ONOS2cli.feature_install("onos-app-metrics")
- main.ONOS3cli.feature_install("onos-app-metrics")
-
utilities.assert_equals(expect=main.TRUE,
actual = cell_file_result and cell_apply_result and\
verify_cell_result and checkout_result and\
@@ -112,10 +111,25 @@
install_time = main.params['JSON']['installedTime']
wdRequest_time = main.params['JSON']['wdRequestTime']
withdrawn_time = main.params['JSON']['withdrawnTime']
+
+ intent_add_lat_list = []
+
+ #Assign 'linear' switch format for basic intent testing
+ main.Mininet1.assign_sw_controller(
+ sw="1", ip1=ONOS1_ip,port1=default_sw_port)
+ main.Mininet1.assign_sw_controller(
+ sw="2", ip1=ONOS2_ip,port1=default_sw_port)
+ main.Mininet1.assign_sw_controller(
+ sw="3", ip1=ONOS2_ip,port1=default_sw_port)
+ main.Mininet1.assign_sw_controller(
+ sw="4", ip1=ONOS2_ip,port1=default_sw_port)
+ main.Mininet1.assign_sw_controller(
+ sw="5", ip1=ONOS3_ip,port1=default_sw_port)
+
+ time.sleep(10)
devices_json_str = main.ONOS1cli.devices()
devices_json_obj = json.loads(devices_json_str)
-
device_id_list = []
#Obtain device id list in ONOS format.
@@ -123,14 +137,11 @@
for device in devices_json_obj:
device_id_list.append(device['id'])
- intent_add_lat_list = []
-
for i in range(0, int(num_iter)):
- #add_point_intent(ingr_device, ingr_port,
- # egr_device, egr_port)
+ #add_point_intent(ingr_device, egr_device,
+ # ingr_port, egr_port)
main.ONOS1cli.add_point_intent(
- device_id_list[0], 2,
- device_id_list[2], 1)
+ device_id_list[0]+"/1", device_id_list[4]+"/1")
#Allow some time for intents to propagate
time.sleep(5)
@@ -213,6 +224,175 @@
def CASE3(self, main):
'''
- CASE3 coming soon
+ Intent Reroute latency
'''
+ import time
+ import json
+ import requests
+ import os
+
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS_user = main.params['CTRL']['user']
+
+ default_sw_port = main.params['CTRL']['port1']
+
+ #number of iterations of case
+ num_iter = main.params['TEST']['numIter']
+
+ #Timestamp keys for json metrics output
+ submit_time = main.params['JSON']['submittedTime']
+ install_time = main.params['JSON']['installedTime']
+ wdRequest_time = main.params['JSON']['wdRequestTime']
+ withdrawn_time = main.params['JSON']['withdrawnTime']
+
+ devices_json_str = main.ONOS1cli.devices()
+ devices_json_obj = json.loads(devices_json_str)
+
+ device_id_list = []
+
+ #Obtain device id list in ONOS format.
+ #They should already be in order (1,2,3,10,11,12,13, etc)
+ for device in devices_json_obj:
+ device_id_list.append(device['id'])
+
+ intent_reroute_lat_list = []
+
+ print device_id_list
+
+ for i in range(0, int(num_iter)):
+ #add_point_intent(ingr_device, ingr_port,
+ # egr_device, egr_port)
+ main.ONOS1cli.add_point_intent(
+ device_id_list[0]+"/2", device_id_list[4]+"/1")
+
+ time.sleep(5)
+
+ intents_str = main.ONOS1cli.intents(json_format=True)
+ intents_obj = json.loads(intents_str)
+ for intent in intents_obj:
+ if intent['state'] == "INSTALLED":
+ main.log.info("Intent installed successfully")
+ intent_id = intent['id']
+ else:
+ #TODO: Add error handling
+ main.log.info("Intent installation failed")
+ intent_id = ""
+
+ #NOTE: this interface is specific to
+ # topo-intentFlower.py topology
+ # reroute case.
+ main.log.info("Disabling interface s2-eth3 <--> s4")
+ main.Mininet1.handle.sendline(
+ "sh ifconfig s2-eth3 down")
+ t0_system = time.time()*1000
+
+ #TODO: Check for correct intent reroute
+ time.sleep(5)
+
+ #Obtain metrics from ONOS 1, 2, 3
+ intents_json_str_1 = main.ONOS1cli.intents_events_metrics()
+ intents_json_str_2 = main.ONOS2cli.intents_events_metrics()
+ intents_json_str_3 = main.ONOS3cli.intents_events_metrics()
+
+ intents_json_obj_1 = json.loads(intents_json_str_1)
+ intents_json_obj_2 = json.loads(intents_json_str_2)
+ intents_json_obj_3 = json.loads(intents_json_str_3)
+
+ #Parse values from the json object
+ intent_install_1 = \
+ intents_json_obj_1[install_time]['value']
+ intent_install_2 = \
+ intents_json_obj_2[install_time]['value']
+ intent_install_3 = \
+ intents_json_obj_3[install_time]['value']
+
+ intent_reroute_lat_1 = \
+ int(intent_install_1) - int(t0_system)
+ intent_reroute_lat_2 = \
+ int(intent_install_2) - int(t0_system)
+ intent_reroute_lat_3 = \
+ int(intent_install_3) - int(t0_system)
+
+ intent_reroute_lat_avg = \
+ (intent_reroute_lat_1 +
+ intent_reroute_lat_2 +
+ intent_reroute_lat_3 ) / 3
+
+ main.log.info("Intent reroute latency avg for iteration "+
+ str(i)+": "+str(intent_reroute_lat_avg))
+
+ if intent_reroute_lat_avg > 0.0 and \
+ intent_reroute_lat_avg < 1000:
+ intent_reroute_lat_list.append(intent_reroute_lat_avg)
+ else:
+ main.log.info("Intent reroute latency exceeded "+
+ "threshold. Skipping iteration "+str(i))
+
+ #TODO: Possibly put this in the driver function
+ main.log.info("Removing intents for next iteration")
+
+ #NOTE: TODO: Currently, the remove intent will
+ # not trigger the intent request
+ # timestamp. Thus we cannot use the same
+ # intent to get the latency over iterations.
+ # we can 1) install different intents every
+ # time, or 2) look into state machine and
+ # determine what timestsamp to get
+ main.ONOS1cli.remove_intent(intent_id)
+
+ #TODO: Report framework
+ print intent_reroute_lat_list
+
+
+ def CASE4(self, main):
+ import time
+ import json
+ import requests
+ import os
+
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS_user = main.params['CTRL']['user']
+
+ default_sw_port = main.params['CTRL']['port1']
+
+ batch_intent_size = main.params['TEST']['batchIntentSize']
+
+ #number of iterations of case
+ num_iter = main.params['TEST']['numIter']
+
+ main.Mininet1.assign_sw_controller(
+ sw="1", ip1=ONOS1_ip,port1=default_sw_port)
+ main.Mininet1.assign_sw_controller(
+ sw="2", ip1=ONOS2_ip,port1=default_sw_port)
+ main.Mininet1.assign_sw_controller(
+ sw="3", ip1=ONOS2_ip,port1=default_sw_port)
+ main.Mininet1.assign_sw_controller(
+ sw="4", ip1=ONOS2_ip,port1=default_sw_port)
+ main.Mininet1.assign_sw_controller(
+ sw="5", ip1=ONOS3_ip,port1=default_sw_port)
+
+ main.log.report("Batch intent installation test of "+
+ str(batch_intent_size) +" intents")
+
+ main.log.info("Getting list of available devices")
+ device_id_list = []
+ json_str = main.ONOS1cli.devices()
+ json_obj = json.loads(json_str)
+ for device in json_obj:
+ device_id_list.append(device['id'])
+
+ for i in range(0, int(num_iter)):
+ main.log.info("Pushing "+batch_intent_size+" intents")
+
+ batch_result = main.ONOS1cli.push_test_intents(
+ "of:0000000000000001/1", "of:0000000000000005/2",
+ batch_intent_size)
+
+ time.sleep(5)
+
+
diff --git a/TestON/tests/IntentPerfNext/IntentPerfNext.topo b/TestON/tests/IntentPerfNext/IntentPerfNext.topo
index fbde0e1..75cb259 100644
--- a/TestON/tests/IntentPerfNext/IntentPerfNext.topo
+++ b/TestON/tests/IntentPerfNext/IntentPerfNext.topo
@@ -56,7 +56,7 @@
<arg1> --custom topo-intentFlower.py </arg1>
<arg2> --arp --mac --topo mytopo</arg2>
<arg3> </arg3>
- <controller> remote,ip=10.128.174.1 </controller>
+ <controller> remote </controller>
</COMPONENTS>
</Mininet1>
diff --git a/TestON/tests/MultiProd13/MultiProd13.params b/TestON/tests/MultiProd13/MultiProd13.params
index 2f99555..b60f5cf 100755
--- a/TestON/tests/MultiProd13/MultiProd13.params
+++ b/TestON/tests/MultiProd13/MultiProd13.params
@@ -1,6 +1,6 @@
<PARAMS>
- <testcases>1,4,5,6,7,8,9</testcases>
+ <testcases>1,4,10,31</testcases>
#Environment variables
<ENV>
diff --git a/TestON/tests/MultiProd13/MultiProd13.py b/TestON/tests/MultiProd13/MultiProd13.py
index 1c989c0..d825c62 100755
--- a/TestON/tests/MultiProd13/MultiProd13.py
+++ b/TestON/tests/MultiProd13/MultiProd13.py
@@ -223,9 +223,9 @@
devices1 = main.ONOScli1.devices()
devices2 = main.ONOScli2.devices()
devices3 = main.ONOScli3.devices()
- print "devices1 = ", devices1
- print "devices2 = ", devices2
- print "devices3 = ", devices3
+ #print "devices1 = ", devices1
+ #print "devices2 = ", devices2
+ #print "devices3 = ", devices3
hosts1 = main.ONOScli1.hosts()
hosts2 = main.ONOScli2.hosts()
hosts3 = main.ONOScli3.hosts()
@@ -355,14 +355,33 @@
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
+
+
+
+ def CASE10(self):
+ main.log.report("This testcase uninstalls the reactive forwarding app")
+ main.log.report("__________________________________")
+ main.case("Uninstalling reactive forwarding app")
+ #Unistall onos-app-fwd app to disable reactive forwarding
+ appUninstall_result1 = main.ONOScli1.feature_uninstall("onos-app-fwd")
+ appUninstall_result2 = main.ONOScli2.feature_uninstall("onos-app-fwd")
+ appUninstall_result3 = main.ONOScli3.feature_uninstall("onos-app-fwd")
+ main.log.info("onos-app-fwd uninstalled")
+
+ #After reactive forwarding is disabled, the reactive flows on switches timeout in 10-15s
+ #So sleep for 15s
+ time.sleep(15)
+
+ case10_result = appUninstall_result1 and appUninstall_result2 and appUninstall_result3
+ utilities.assert_equals(expect=main.TRUE, actual=case10_result,onpass="Reactive forwarding app uninstallation successful",onfail="Reactive forwarding app uninstallation failed")
+
+
def CASE6(self):
main.log.report("This testcase is testing the addition of host intents and then doing pingall")
main.log.report("__________________________________")
- main.case("Uninstalling reactive forwarding app and addhost intents")
+ main.case("Obtaining hostsfor adding host intents")
main.step("Get hosts")
- main.ONOScli1.handle.sendline("hosts")
- main.ONOScli1.handle.expect("onos>")
- hosts = main.ONOScli1.handle.before
+ hosts = main.ONOScli1.hosts()
main.log.info(hosts)
main.step("Get all devices id")
@@ -371,7 +390,8 @@
#ONOS displays the hosts in hex format unlike mininet which does in decimal format
#So take care while adding intents
-
+
+ '''
main.step("Add host intents for mn hosts(h8-h18,h9-h19,h10-h20,h11-h21,h12-h22,h13-h23,h14-h24,h15-h25,h16-h26,h17-h27)")
hth_intent_result = main.ONOScli1.add_host_intent("00:00:00:00:00:08/-1", "00:00:00:00:00:12/-1")
hth_intent_result = main.ONOScli1.add_host_intent("00:00:00:00:00:09/-1", "00:00:00:00:00:13/-1")
@@ -383,16 +403,17 @@
hth_intent_result = main.ONOScli1.add_host_intent("00:00:00:00:00:0F/-1", "00:00:00:00:00:19/-1")
hth_intent_result = main.ONOScli1.add_host_intent("00:00:00:00:00:10/-1", "00:00:00:00:00:1A/-1")
hth_intent_result = main.ONOScli1.add_host_intent("00:00:00:00:00:11/-1", "00:00:00:00:00:1B/-1")
+ '''
-
- #Unistall onos-app-fwd app to disable reactive forwarding
- main.step("Unistall onos-app-fwd app to disable reactive forwarding")
- appUninstall_result = main.ONOScli1.feature_uninstall("onos-app-fwd")
- main.log.info("onos-app-fwd uninstalled")
-
- #After reactive forwarding is disabled, the reactive flows on switches timeout in 10-15s
- #So sleep for 15s
- time.sleep(15)
+ for i in range(8,18):
+ main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+ host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+ host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+ #NOTE: get host can return None
+ #TODO: handle this
+ host1_id = main.ONOScli1.get_host(host1)['id']
+ host2_id = main.ONOScli1.get_host(host2)['id']
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
flowHandle = main.ONOScli1.flows()
#print "flowHandle = ", flowHandle
@@ -634,7 +655,7 @@
def CASE8(self):
'''
- Host intents removal
+ Intent removal
'''
main.log.report("This testcase removes host intents before adding the point intents")
main.log.report("__________________________________")
@@ -643,31 +664,37 @@
main.step("Obtain the intent id's")
intent_result = main.ONOScli1.intents()
#print "intent_result = ",intent_result
+
intent_linewise = intent_result.split("\n")
- intent_linewise = intent_linewise[1:-1] #ignore the first and last item of the list obtained from split
- #for line in intent_linewise:
- #print "line = ", line
- intentids = []
+ intentList = []
for line in intent_linewise:
+ if line.startswith("id="):
+ intentList.append(line)
+
+ intentids = []
+ for line in intentList:
intentids.append(line.split(",")[0].split("=")[1])
for id in intentids:
print "id = ", id
-
+
main.step("Iterate through the intentids list and remove each intent")
for id in intentids:
main.ONOScli1.remove_intent(intent_id = id)
-
+
intent_result = main.ONOScli1.intents()
intent_linewise = intent_result.split("\n")
- intent_linewise = intent_linewise[1:-1] #ignore the first and last item of the list obtained from split
-
+ list_afterRemoval = []
+ for line in intent_linewise:
+ if line.startswith("id="):
+ list_afterRemoval.append(line)
+
intentState = {}
- for id, line in zip(intentids, intent_linewise):
+ for id, line in zip(intentids, list_afterRemoval):
#print "line after removing intent = ", line
x = line.split(",")
state = x[1].split("=")[1]
intentState[id] = state
-
+
case8_result = main.TRUE
for key,value in intentState.iteritems():
print "key,value = ", key, value
@@ -676,12 +703,36 @@
else:
case8_result = case8_result and main.FALSE
- if case8_result == main.TRUE:
+ i = 8
+ Ping_Result = main.TRUE
+ while i <18 :
+ main.log.info("\n\nh"+str(i)+" is Pinging h" + str(i+10))
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ if ping==main.TRUE:
+ Ping_Result = main.TRUE
+ elif ping==main.FALSE:
+ i+=1
+ Ping_Result = main.FALSE
+ else:
+ main.log.info("Unknown error")
+ Ping_Result = main.ERROR
+
+ #Note: If the ping result failed, that means the intents have been withdrawn correctly.
+ if Ping_Result==main.TRUE:
+ main.log.report("Host intents have not been withdrawn correctly")
+ #main.cleanup()
+ #main.exit()
+ if Ping_Result==main.FALSE:
+ main.log.report("Host intents have been withdrawn correctly")
+
+ case8_result = case8_result and Ping_Result
+
+ if case8_result == main.FALSE:
main.log.report("Intent removal successful")
else:
main.log.report("Intent removal failed")
- utilities.assert_equals(expect=main.TRUE, actual=case8_result,
+ utilities.assert_equals(expect=main.FALSE, actual=case8_result,
onpass="Intent removal test successful",
onfail="Intent removal test failed")
@@ -867,3 +918,34 @@
onpass="Ping all test after Point intents addition successful",
onfail="Ping all test after Point intents addition failed")
+ def CASE31(self):
+ '''
+ This test case adds point intent related to SDN-IP matching on ICMP (ethertype=IPV4, ipProto=1)
+ '''
+ import json
+
+ main.log.report("This test case adds point intent related to SDN-IP matching on ICMP")
+ main.step("Adding bidirectional point intent")
+ #add-point-intent --ipSrc=10.0.0.8/32 --ipDst=10.0.0.18/32 --ethType=IPV4 --ipProto=1 of:0000000000003008/1 of:0000000000006018/1
+
+ hosts_json = json.loads(main.ONOScli1.hosts())
+ for i in range(8,11):
+ main.log.info("Adding point intent between h"+str(i)+" and h"+str(i+10))
+ host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+ host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+ host1_id = main.ONOScli1.get_host(host1)['id']
+ host2_id = main.ONOScli1.get_host(host2)['id']
+ for host in hosts_json:
+ if host['id'] == host1_id:
+ ip1 = host['ips']+"/"+"32"
+ device1 = str(host['location']['device'])
+ port1 = 1
+ print "ip1 = ", ip1
+ print "device1 = ", device1
+ elif host['id'] == host2_id:
+ ip2 = str(host['ips'])+"/"+"32"
+ device2 = str(host['location']["device"])
+ port2 = 1
+ print "ip2 = ", ip2
+ print "device2 = ", device2
+
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
new file mode 100644
index 0000000..e1e75f9
--- /dev/null
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
@@ -0,0 +1,63 @@
+<PARAMS>
+ <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <ENV>
+ <cellName>HA</cellName>
+ </ENV>
+ <Git>True</Git>
+
+ <CTRL>
+ <ip1>10.128.30.11</ip1>
+ <port1>6633</port1>
+
+ <ip2>10.128.30.12</ip2>
+ <port2>6633</port2>
+
+ <ip3>10.128.30.13</ip3>
+ <port3>6633</port3>
+
+ <ip4>10.128.30.14</ip4>
+ <port4>6633</port4>
+
+ <ip5>10.128.30.15</ip5>
+ <port5>6633</port5>
+
+ <ip6>10.128.30.16</ip6>
+ <port6>6633</port6>
+
+ <ip7>10.128.30.17</ip7>
+ <port7>6633</port7>
+ </CTRL>
+ <TESTONUSER>admin</TESTONUSER>
+ <TESTONIP>10.128.30.10</TESTONIP>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+ <timers>
+ <LinkDiscovery>2</LinkDiscovery>
+ <SwitchDiscovery>2</SwitchDiscovery>
+ </timers>
+ <MNtcpdump>
+ <intf>eth0</intf>
+ <port> </port>
+ <folder>~/packet_captures/</folder>
+ </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
new file mode 100644
index 0000000..e6f2fde
--- /dev/null
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
@@ -0,0 +1,814 @@
+'''
+Description: This test is to determine if a single
+ instance ONOS 'cluster' can handle a restart
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE6: The Failure case. Since this is the Sanity test, we do nothing.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+'''
+class SingleInstanceHATestRestart:
+
+ def __init__(self) :
+ self.default = ''
+
+ def CASE1(self,main) :
+ '''
+ CASE1 is to compile ONOS and push it to the test machines
+
+ Startup sequence:
+ git pull
+ mvn clean install
+ onos-package
+ cell <name>
+ onos-verify-cell
+ NOTE: temporary - onos-remove-raft-logs
+ onos-install -f
+ onos-wait-for-start
+ '''
+ import time
+ main.log.report("ONOS Single node cluster restart HA test - initialization")
+ main.case("Setting up test environment")
+
+ # load some vairables from the params file
+ PULL_CODE = False
+ if main.params['Git'] == 'True':
+ PULL_CODE = True
+ cell_name = main.params['ENV']['cellName']
+
+ #set global variables
+ global ONOS1_ip
+ global ONOS1_port
+ global ONOS2_ip
+ global ONOS2_port
+ global ONOS3_ip
+ global ONOS3_port
+ global ONOS4_ip
+ global ONOS4_port
+ global ONOS5_ip
+ global ONOS5_port
+ global ONOS6_ip
+ global ONOS6_port
+ global ONOS7_ip
+ global ONOS7_port
+
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS1_port = main.params['CTRL']['port1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS2_port = main.params['CTRL']['port2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS3_port = main.params['CTRL']['port3']
+ ONOS4_ip = main.params['CTRL']['ip4']
+ ONOS4_port = main.params['CTRL']['port4']
+ ONOS5_ip = main.params['CTRL']['ip5']
+ ONOS5_port = main.params['CTRL']['port5']
+ ONOS6_ip = main.params['CTRL']['ip6']
+ ONOS6_port = main.params['CTRL']['port6']
+ ONOS7_ip = main.params['CTRL']['ip7']
+ ONOS7_port = main.params['CTRL']['port7']
+
+
+ main.step("Applying cell variable to environment")
+ cell_result = main.ONOSbench.set_cell(cell_name)
+ verify_result = main.ONOSbench.verify_cell()
+
+ #FIXME:this is short term fix
+ main.log.report("Removing raft logs")
+ main.ONOSbench.onos_remove_raft_logs()
+ main.log.report("Uninstalling ONOS")
+ main.ONOSbench.onos_uninstall(ONOS1_ip)
+ main.ONOSbench.onos_uninstall(ONOS2_ip)
+ main.ONOSbench.onos_uninstall(ONOS3_ip)
+ main.ONOSbench.onos_uninstall(ONOS4_ip)
+ main.ONOSbench.onos_uninstall(ONOS5_ip)
+ main.ONOSbench.onos_uninstall(ONOS6_ip)
+ main.ONOSbench.onos_uninstall(ONOS7_ip)
+
+ clean_install_result = main.TRUE
+ git_pull_result = main.TRUE
+
+ main.step("Compiling the latest version of ONOS")
+ if PULL_CODE:
+ main.step("Git checkout and pull master")
+ main.ONOSbench.git_checkout("master")
+ git_pull_result = main.ONOSbench.git_pull()
+
+ main.step("Using mvn clean & install")
+ clean_install_result = main.TRUE
+ if git_pull_result == main.TRUE:
+ clean_install_result = main.ONOSbench.clean_install()
+ else:
+ main.log.warn("Did not pull new code so skipping mvn "+ \
+ "clean install")
+ main.ONOSbench.get_version(report=True)
+
+ main.step("Creating ONOS package")
+ package_result = main.ONOSbench.onos_package()
+
+ main.step("Installing ONOS package")
+ onos1_install_result = main.ONOSbench.onos_install(options="-f",
+ node=ONOS1_ip)
+
+
+ main.step("Checking if ONOS is up yet")
+ #TODO: Refactor
+ # check bundle:list?
+ #this should be enough for ONOS to start
+ time.sleep(60)
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+ # TODO: if it becomes an issue, we can retry this step a few times
+
+
+ cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+
+ main.step("Start Packet Capture MN")
+ main.Mininet2.start_tcpdump(
+ str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+ intf = main.params['MNtcpdump']['intf'],
+ port = main.params['MNtcpdump']['port'])
+
+
+ case1_result = (clean_install_result and package_result and
+ cell_result and verify_result and onos1_install_result and
+ onos1_isup and cli1_results)
+
+ utilities.assert_equals(expect=main.TRUE, actual=case1_result,
+ onpass="Test startup successful",
+ onfail="Test startup NOT successful")
+
+
+ if case1_result==main.FALSE:
+ main.cleanup()
+ main.exit()
+
+ def CASE2(self,main) :
+ '''
+ Assign mastership to controllers
+ '''
+ import time
+ import json
+ import re
+
+ main.log.report("Assigning switches to controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign switches to controllers")
+
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),
+ ip1=ONOS1_ip,port1=ONOS1_port)
+
+ mastership_check = main.TRUE
+ for i in range (1,29):
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ try:
+ main.log.info(str(response))
+ except:
+ main.log.info(repr(response))
+ if re.search("tcp:"+ONOS1_ip,response):
+ mastership_check = mastership_check and main.TRUE
+ else:
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Switch mastership assigned correctly")
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership assigned correctly",
+ onfail="Switches not assigned correctly to controllers")
+
+ #TODO: If assign roles is working reliably then manually
+ # assign mastership to the controller we want
+
+
+ def CASE3(self,main) :
+ """
+ Assign intents
+
+ """
+ import time
+ import json
+ import re
+ main.log.report("Adding host intents")
+ main.case("Adding host Intents")
+
+ main.step("Discovering Hosts( Via pingall for now)")
+ #FIXME: Once we have a host discovery mechanism, use that instead
+
+ #REACTIVE FWD test
+ ping_result = main.FALSE
+ time1 = time.time()
+ ping_result = main.Mininet1.pingall()
+ time2 = time.time()
+ main.log.info("Time for pingall: %2f seconds" % (time2 - time1))
+
+ #uninstall onos-app-fwd
+ main.log.info("Uninstall reactive forwarding app")
+ main.ONOScli1.feature_uninstall("onos-app-fwd")
+
+ main.step("Add host intents")
+ #TODO: move the host numbers to params
+ import json
+ intents_json= json.loads(main.ONOScli1.hosts())
+ intent_add_result = main.FALSE
+ for i in range(8,18):
+ main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
+ host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
+ host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
+ #NOTE: get host can return None
+ #TODO: handle this
+ host1_id = main.ONOScli1.get_host(host1)['id']
+ host2_id = main.ONOScli1.get_host(host2)['id']
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ intent_add_result = intent_add_result and tmp_result
+ #TODO Check if intents all exist in datastore
+ #NOTE: Do we need to print this once the test is working?
+ #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
+ # sort_keys=True, indent=4, separators=(',', ': ') ) )
+
+ def CASE4(self,main) :
+ """
+ Ping across added host intents
+ """
+ description = " Ping across added host intents"
+ main.log.report(description)
+ main.case(description)
+ Ping_Result = main.TRUE
+ for i in range(8,18):
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ Ping_Result = Ping_Result and ping
+ if ping==main.FALSE:
+ main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10))
+ elif ping==main.TRUE:
+ main.log.info("Ping test passed!")
+ Ping_Result = main.TRUE
+ if Ping_Result==main.FALSE:
+ main.log.report("Intents have not been installed correctly, pings failed.")
+ if Ping_Result==main.TRUE:
+ main.log.report("Intents have been installed correctly and verified by pings")
+ utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
+ onpass="Intents have been installed correctly and pings work",
+ onfail ="Intents have not been installed correctly, pings failed." )
+
+ def CASE5(self,main) :
+ '''
+ Reading state of ONOS
+ '''
+ import time
+ import json
+ from subprocess import Popen, PIPE
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+
+ main.log.report("Setting up and gathering data for current state")
+ main.case("Setting up and gathering data for current state")
+ #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node
+ #We can then compare them with eachother and also with past states
+
+ main.step("Get the Mastership of each switch from each controller")
+ global mastership_state
+ ONOS1_mastership = main.ONOScli1.roles()
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ #TODO: Make this a meaningful check
+ if "Error" in ONOS1_mastership or not ONOS1_mastership:
+ main.log.report("Error in getting ONOS roles")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ consistent_mastership = main.FALSE
+ else:
+ mastership_state = ONOS1_mastership
+ consistent_mastership = main.TRUE
+ main.log.report("Switch roles are consistent across all ONOS nodes")
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+ onpass="Switch roles are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of switch roles")
+
+
+ main.step("Get the intents from each controller")
+ global intent_state
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ else:
+ intent_check = main.TRUE
+ main.log.report("Intents are consistent across all ONOS nodes")
+ utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+ onpass="Intents are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of intents")
+
+
+ main.step("Get the flows from each controller")
+ global flow_state
+ ONOS1_flows = main.ONOScli1.flows( json_format=True )
+ flow_check = main.FALSE
+ if "Error" in ONOS1_flows or not ONOS1_flows:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows)
+ else:
+ #TODO: Do a better check, maybe compare flows on switches?
+ flow_state = ONOS1_flows
+ flow_check = main.TRUE
+ main.log.report("Flow count is consistent across all ONOS nodes")
+ utilities.assert_equals(expect = main.TRUE,actual=flow_check,
+ onpass="The flow count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different flow counts")
+
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,29):
+ flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+
+ #TODO: Compare switch flow tables with ONOS flow tables
+
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],
+ target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],
+ target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],
+ target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],
+ target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],
+ target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],
+ target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],
+ target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],
+ target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],
+ target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],
+ target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ temp = ()
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ for controller in range(1): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+
+ topo_result = devices_results and ports_results and links_results
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+
+ final_assert = main.TRUE
+ final_assert = final_assert and topo_result and flow_check \
+ and intent_check and consistent_mastership
+ utilities.assert_equals(expect=main.TRUE, actual=final_assert,
+ onpass="State check successful",
+ onfail="State check NOT successful")
+
+
+ def CASE6(self,main) :
+ '''
+ The Failure case.
+ '''
+ import time
+
+ main.log.report("Restart ONOS node")
+ main.log.case("Restart ONOS node")
+ main.ONOSbench.onos_kill(ONOS1_ip)
+ start = time.time()
+
+ main.step("Checking if ONOS is up yet")
+ count = 0
+ while count < 10
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if onos1_isup == main.TRUE:
+ elapsed = time.time() - start
+ break
+ else:
+ count = count + 1
+
+ cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
+
+ case_results = main.TRUE and onosi1_isup and cli_result
+ utilities.assert_equals(expect=main.TRUE, actual=case_results,
+ onpass="ONOS restart successful",
+ onfail="ONOS restart NOT successful")
+ main.log.info("ONOS took %s seconds to restart" % str(elapsed) )
+
+ def CASE7(self,main) :
+ '''
+ Check state after ONOS failure
+ '''
+ import os
+ import json
+ main.case("Running ONOS Constant State Tests")
+
+ main.step("Check if switch roles are consistent across all nodes")
+ ONOS1_mastership = main.ONOScli1.roles()
+ #FIXME: Refactor this whole case for single instance
+ #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
+ if "Error" in ONOS1_mastership or not ONOS1_mastership:
+ main.log.report("Error in getting ONOS mastership")
+ main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
+ consistent_mastership = main.FALSE
+ else:
+ consistent_mastership = main.TRUE
+ main.log.report("Switch roles are consistent across all ONOS nodes")
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
+ onpass="Switch roles are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of switch roles")
+
+
+ description2 = "Compare switch roles from before failure"
+ main.step(description2)
+
+ current_json = json.loads(ONOS1_mastership)
+ old_json = json.loads(mastership_state)
+ mastership_check = main.TRUE
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+
+ current = [switch['master'] for switch in current_json if switchDPID in switch['id']]
+ old = [switch['master'] for switch in old_json if switchDPID in switch['id']]
+ if current == old:
+ mastership_check = mastership_check and main.TRUE
+ else:
+ main.log.warn("Mastership of switch %s changed" % switchDPID)
+ mastership_check = main.FALSE
+ if mastership_check == main.TRUE:
+ main.log.report("Mastership of Switches was not changed")
+ utilities.assert_equals(expect=main.TRUE,actual=mastership_check,
+ onpass="Mastership of Switches was not changed",
+ onfail="Mastership of some switches changed")
+ mastership_check = mastership_check and consistent_mastership
+
+
+
+ main.step("Get the intents and compare across all nodes")
+ ONOS1_intents = main.ONOScli1.intents( json_format=True )
+ intent_check = main.FALSE
+ if "Error" in ONOS1_intents or not ONOS1_intents:
+ main.log.report("Error in getting ONOS intents")
+ main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
+ else:
+ intent_state = ONOS1_intents
+ intent_check = main.TRUE
+ main.log.report("Intents are consistent across all ONOS nodes")
+ utilities.assert_equals(expect = main.TRUE,actual=intent_check,
+ onpass="Intents are consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of intents")
+
+ main.step("Compare current intents with intents before the failure")
+ if intent_state == ONOS1_intents:
+ same_intents = main.TRUE
+ main.log.report("Intents are consistent with before failure")
+ #TODO: possibly the states have changed? we may need to figure out what the aceptable states are
+ else:
+ same_intents = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=same_intents,
+ onpass="Intents are consistent with before failure",
+ onfail="The Intents changed during failure")
+ intent_check = intent_check and same_intents
+
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ Flow_Tables = main.TRUE
+ flows2=[]
+ for i in range(28):
+ main.log.info("Checking flow table on s" + str(i+1))
+ tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ flows2.append(tmp_flows)
+ Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ if Flow_Tables == main.FALSE:
+ main.log.info("Differences in flow table for switch: "+str(i+1))
+ break
+ if Flow_Tables == main.TRUE:
+ main.log.report("No changes were found in the flow tables")
+ utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
+ onpass="No changes were found in the flow tables",
+ onfail="Changes were found in the flow tables")
+
+ main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+ #FIXME: This check is always failing. Investigate cause
+ #NOTE: this may be something to do with file permsissions
+ # or slight change in format
+ main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
+ Loss_In_Pings = main.FALSE
+ #NOTE: checkForLoss returns main.FALSE with 0% packet loss
+ for i in range(8,18):
+ main.log.info("Checking for a loss in pings along flow from s" + str(i))
+ Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if Loss_In_Pings == main.TRUE:
+ main.log.info("Loss in ping detected")
+ elif Loss_In_Pings == main.ERROR:
+ main.log.info("There are multiple mininet process running")
+ elif Loss_In_Pings == main.FALSE:
+ main.log.info("No Loss in the pings")
+ main.log.report("No loss of dataplane connectivity")
+ utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
+ onpass="No Loss of connectivity",
+ onfail="Loss of dataplane connectivity detected")
+
+
+ #TODO:add topology to this or leave as a seperate case?
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = int(result)
+ if result == main.TRUE:
+ main.log.report("Constant State Tests Passed")
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Constant State Tests Passed",
+ onfail="Constant state tests failed")
+
+ def CASE8 (self,main):
+ '''
+ Compare topo
+ '''
+ import sys
+ sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params
+ from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH
+ import json
+ import time
+
+ description ="Compare ONOS Topology view to Mininet topology"
+ main.case(description)
+ main.log.report(description)
+ main.step("Create TestONTopology object")
+ ctrls = []
+ count = 1
+ temp = ()
+ temp = temp + (getattr(main,('ONOS' + str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+
+ main.step("Comparing ONOS topology to MN")
+ devices_results = main.TRUE
+ ports_results = main.TRUE
+ links_results = main.TRUE
+ topo_result = main.FALSE
+ start_time = time.time()
+ elapsed = 0
+ count = 0
+ while topo_result == main.FALSE and elapsed < 120:
+ count = count + 1
+ try:
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ for controller in range(1): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
+ except:
+ main.log.error("something went wrong in topo comparison")
+ main.log.warn( repr( devices ) )
+ main.log.warn( repr( ports ) )
+ main.log.warn( repr( links ) )
+
+ devices_results = devices_results and current_devices_result
+ ports_results = ports_results and current_ports_result
+ links_results = links_results and current_links_result
+ elapsed = time.time() - start_time
+ time_threshold = elapsed < 1
+ topo_result = devices_results and ports_results and links_results and time_threshold
+ #TODO make sure this step is non-blocking. IE add a timeout
+ main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ str(elapsed) + " seconds, " + str(count) +" tries" )
+ utilities.assert_equals(expect=main.TRUE, actual=topo_result,
+ onpass="Topology Check Test successful",
+ onfail="Topology Check Test NOT successful")
+ if topo_result == main.TRUE:
+ main.log.report("ONOS topology view matches Mininet topology")
+
+
+ def CASE9 (self,main):
+ '''
+ Link s3-s28 down
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+ description = "Turn off a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+
+ main.step("Kill Link between s3 and s28")
+ Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Down,
+ onpass="Link down succesful",
+ onfail="Failed to bring link down")
+ #TODO do some sort of check here
+
+ def CASE10 (self,main):
+ '''
+ Link s3-s28 up
+ '''
+ #NOTE: You should probably run a topology check after this
+
+ link_sleep = int(main.params['timers']['LinkDiscovery'])
+
+ description = "Restore a link to ensure that Link Discovery is working properly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Bring link between s3 and s28 back up")
+ Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+ main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered")
+ time.sleep(link_sleep)
+ utilities.assert_equals(expect=main.TRUE,actual=Link_Up,
+ onpass="Link up succesful",
+ onfail="Failed to bring link up")
+ #TODO do some sort of check here
+
+
+ def CASE11 (self, main) :
+ '''
+ Switch Down
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+
+ switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+ description = "Killing a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ #TODO: Make this switch parameterizable
+ main.step("Kill s28 ")
+ main.log.report("Deleting s28")
+ #FIXME: use new dynamic topo functions
+ main.Mininet1.del_switch("s28")
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+ time.sleep(switch_sleep)
+ #Peek at the deleted switch
+ main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+ #TODO do some sort of check here
+
+ def CASE12 (self, main) :
+ '''
+ Switch Up
+ '''
+ #NOTE: You should probably run a topology check after this
+ import time
+ #FIXME: use new dynamic topo functions
+ description = "Adding a switch to ensure it is discovered correctly"
+ main.log.report(description)
+ main.case(description)
+
+ main.step("Add back s28")
+ main.log.report("Adding back s28")
+ main.Mininet1.add_switch("s28", dpid = '0000000000002800')
+ #TODO: New dpid or same? Ask Thomas?
+ main.Mininet1.add_link('s28', 's3')
+ main.Mininet1.add_link('s28', 's6')
+ main.Mininet1.add_link('s28', 'h28')
+ main.Mininet1.assign_sw_controller(sw="28",
+ ip1=ONOS1_ip,port1=ONOS1_port)
+ main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
+ time.sleep(switch_sleep)
+ #Peek at the added switch
+ main.log.warn(main.ONOScli1.get_device(dpid="0028"))
+ #TODO do some sort of check here
+
+ def CASE13 (self, main) :
+ '''
+ Clean up
+ '''
+ import os
+ import time
+ description = "Test Cleanup"
+ main.log.report(description)
+ main.case(description)
+ main.step("Killing tcpdumps")
+ main.Mininet2.stop_tcpdump()
+
+ main.step("Copying MN pcap and ONOS log files to test station")
+ testname = main.TEST
+ #NOTE: MN Pcap file is being saved to ~/packet_captures
+ # scp this file as MN and TestON aren't necessarily the same vm
+ #FIXME: scp
+ #####mn files
+ #TODO: Load these from params
+ #NOTE: must end in /
+ log_folder = "/opt/onos/log/"
+ log_files = ["karaf.log", "karaf.log.1"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+
+ #std*.log's
+ #NOTE: must end in /
+ log_folder = "/opt/onos/var/"
+ log_files = ["stderr.log", "stdout.log"]
+ #NOTE: must end in /
+ dst_dir = "~/packet_captures/"
+ for f in log_files:
+ main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ dst_dir + str(testname) + "-ONOS1-"+f )
+
+
+ #sleep so scp can finish
+ time.sleep(10)
+ main.step("Packing and rotating pcap archives")
+ os.system("~/TestON/dependencies/rotate.sh "+ str(testname))
+
+
+ #TODO: actually check something here
+ utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+ onpass="Test cleanup successful",
+ onfail="Test cleanup NOT successful")
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.topo b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.topo
new file mode 100644
index 0000000..4d4156c
--- /dev/null
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.topo
@@ -0,0 +1,173 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli3>
+
+
+ <ONOScli4>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli4>
+
+
+ <ONOScli5>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli5>
+
+
+ <ONOScli6>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli6>
+
+
+ <ONOScli7>
+ <host>10.128.30.10</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>OnosCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOScli7>
+
+ <ONOS1>
+ <host>10.128.30.11</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>9</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.30.12</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.30.13</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>11</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS3>
+
+ <ONOS4>
+ <host>10.128.30.14</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>12</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS4>
+
+ <ONOS5>
+ <host>10.128.30.15</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS5>
+
+ <ONOS6>
+ <host>10.128.30.16</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS6>
+
+ <ONOS7>
+ <host>10.128.30.17</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>15</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS7>
+
+ <Mininet1>
+ <host>10.128.30.9</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>MininetCliDriver</type>
+ <connect_order>16</connect_order>
+ <COMPONENTS>
+ #Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo </arg2>
+ <arg3> </arg3>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet1>
+
+ <Mininet2>
+ <host>10.128.30.9</host>
+ <user>admin</user>
+ <password>onos_test</password>
+ <type>RemoteMininetDriver</type>
+ <connect_order>17</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet2>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/SingleInstanceHATestRestart/__init__.py b/TestON/tests/SingleInstanceHATestRestart/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/SingleInstanceHATestRestart/__init__.py
diff --git a/TestON/tests/TopoConvNext/TopoConvNext.params b/TestON/tests/TopoConvNext/TopoConvNext.params
index a7a5ac6..96cb9ad 100644
--- a/TestON/tests/TopoConvNext/TopoConvNext.params
+++ b/TestON/tests/TopoConvNext/TopoConvNext.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,3,2,3,2</testcases>
+ <testcases>1,2,3,2,3,2,3,4,2,3,2,3,2,4,2,3,2,3,2</testcases>
<ENV>
<cellName>topo_conv_test</cellName>
@@ -42,10 +42,19 @@
<onosLogFile>/opt/onos/log/karaf*</onosLogFile>
#Number of times to iterate each case
- <numIter>10</numIter>
- <numSwitch>100</numSwitch>
+ <numIter>5</numIter>
+ <numSwitch1>200</numSwitch1>
+ <numSwitch2>400</numSwitch2>
+ <numSwitch3>500</numSwitch3>
#Number of iterations to ignore initially
- <iterIgnore>0</iterIgnore>
+ <iterIgnore>1</iterIgnore>
+
+ <topo_accumulator_config>
+ large_topo_event_accumulator.cfg
+ </topo_accumulator_config>
+ <topo_config_name>
+ org.onlab.onos.net.topology.impl.DefaultTopologyProvider.cfg
+ </topo_config_name>
<swDisc100Threshold>0,100000</swDisc100Threshold>
</TEST>
diff --git a/TestON/tests/TopoConvNext/TopoConvNext.py b/TestON/tests/TopoConvNext/TopoConvNext.py
index 37fb24d..52a951c 100644
--- a/TestON/tests/TopoConvNext/TopoConvNext.py
+++ b/TestON/tests/TopoConvNext/TopoConvNext.py
@@ -27,7 +27,9 @@
#******
#Global cluster count for scale-out purposes
global cluster_count
- cluster_count = 3
+ global topo_iteration
+ topo_iteration = 1
+ cluster_count = 1
#******
cell_name = main.params['ENV']['cellName']
@@ -45,8 +47,18 @@
BENCH_ip = main.params['BENCH']['ip']
main.case("Setting up test environment")
+ main.log.info("copying topology event accumulator config file"+\
+ " to ONOS package/etc/ directory")
+ topo_config_name = main.params['TEST']['topo_config_name']
+ topo_config =\
+ main.params['TEST']['topo_accumulator_config']
+ main.ONOSbench.handle.sendline("cp ~/"+topo_config+\
+ " ~/ONOS/tools/package/etc/"+\
+ topo_config_name)
+ main.ONOSbench.handle.expect("\$")
+
main.log.info("Uninstalling previous instances")
- main.ONOSbench.onos_uninstall(node_ip = ONOS1_ip)
+ #main.ONOSbench.onos_uninstall(node_ip = ONOS1_ip)
main.ONOSbench.onos_uninstall(node_ip = ONOS2_ip)
main.ONOSbench.onos_uninstall(node_ip = ONOS3_ip)
main.ONOSbench.onos_uninstall(node_ip = ONOS4_ip)
@@ -54,21 +66,23 @@
main.ONOSbench.onos_uninstall(node_ip = ONOS6_ip)
main.ONOSbench.onos_uninstall(node_ip = ONOS7_ip)
- main.case("Removing raft logs")
- main.ONOSbench.onos_remove_raft_logs()
-
main.log.report("Setting up test environment")
main.step("Creating cell file")
cell_file_result = main.ONOSbench.create_cell_file(
BENCH_ip, cell_name, MN1_ip,
"onos-core,onos-app-metrics",
- ONOS1_ip, ONOS2_ip, ONOS3_ip)
-
+ #ONOS1_ip, ONOS2_ip, ONOS3_ip)
+ ONOS1_ip)
+
main.step("Applying cell file to environment")
cell_apply_result = main.ONOSbench.set_cell(cell_name)
verify_cell_result = main.ONOSbench.verify_cell()
+ main.step("Removing raft logs")
+ main.ONOSbench.onos_remove_raft_logs()
+ time.sleep(10)
+
main.step("Git checkout and pull "+checkout_branch)
if git_pull == 'on':
checkout_result = \
@@ -79,14 +93,17 @@
pull_result = main.TRUE
main.log.info("Skipped git checkout and pull")
+ main.log.report("Commit information - ")
+ main.ONOSbench.get_version()
+
main.step("Using mvn clean & install")
#mvn_result = main.ONOSbench.clean_install()
mvn_result = main.TRUE
main.step("Set cell for ONOS cli env")
main.ONOS1cli.set_cell(cell_name)
- main.ONOS2cli.set_cell(cell_name)
- main.ONOS3cli.set_cell(cell_name)
+ #main.ONOS2cli.set_cell(cell_name)
+ #main.ONOS3cli.set_cell(cell_name)
main.step("Creating ONOS package")
package_result = main.ONOSbench.onos_package()
@@ -94,15 +111,15 @@
#Start test with single node only
main.step("Installing ONOS package")
install1_result = main.ONOSbench.onos_install(node=ONOS1_ip)
- install2_result = main.ONOSbench.onos_install(node=ONOS2_ip)
- install3_result = main.ONOSbench.onos_install(node=ONOS3_ip)
+ #install2_result = main.ONOSbench.onos_install(node=ONOS2_ip)
+ #install3_result = main.ONOSbench.onos_install(node=ONOS3_ip)
time.sleep(10)
main.step("Start onos cli")
cli1 = main.ONOS1cli.start_onos_cli(ONOS1_ip)
- cli2 = main.ONOS2cli.start_onos_cli(ONOS2_ip)
- cli3 = main.ONOS3cli.start_onos_cli(ONOS3_ip)
+ #cli2 = main.ONOS2cli.start_onos_cli(ONOS2_ip)
+ #cli3 = main.ONOS3cli.start_onos_cli(ONOS3_ip)
main.step("Enable metrics feature")
#main.ONOS1cli.feature_install("onos-app-metrics")
@@ -111,8 +128,8 @@
actual= cell_file_result and cell_apply_result and\
verify_cell_result and checkout_result and\
pull_result and mvn_result and\
- install1_result and install2_result and\
- install3_result,
+ install1_result, #and install2_result and\
+ #install3_result,
onpass="Test Environment setup successful",
onfail="Failed to setup test environment")
@@ -154,7 +171,21 @@
#Number of iterations of case
num_iter = main.params['TEST']['numIter']
- num_sw = main.params['TEST']['numSwitch']
+ iter_ignore = int(main.params['TEST']['iterIgnore'])
+
+ #***********
+ #Global number of switches that change
+ #throughout the test
+ global num_sw
+ global topo_iteration
+ global cluster_count
+ if topo_iteration == 1:
+ num_sw = main.params['TEST']['numSwitch1']
+ elif topo_iteration == 2:
+ num_sw = main.params['TEST']['numSwitch2']
+ elif topo_iteration == 3:
+ num_sw = main.params['TEST']['numSwitch3']
+ #***********
#Timestamp 'keys' for json metrics output.
#These are subject to change, hence moved into params
@@ -171,8 +202,8 @@
sw_discovery_lat_list = []
syn_ack_delta_list = []
- main.case(str(num_sw)+" switch per "+str(cluster_count)+
- " nodes convergence latency")
+ main.case(str(num_sw)+" switches distributed across "+
+ str(cluster_count)+" nodes convergence latency")
main.log.report("Large topology convergence and scale-out test")
main.log.report("Currently active ONOS node(s): ")
@@ -180,16 +211,22 @@
for node in range(1, cluster_count+1):
report_str += (str(node) + " ")
main.log.report(report_str)
-
- main.step("Assigning "+num_sw+" switches to each ONOS")
+ main.log.report("Topology size: "+str(num_sw)+" switches")
+
+ main.step("Distributing "+num_sw+" switches to each ONOS")
index = 1
for node in range(1, cluster_count+1):
- for i in range(index, int(num_sw)+index):
+ for i in range(index, (int(num_sw)/cluster_count)+index):
main.Mininet1.assign_sw_controller(
sw=str(i),
ip1=ONOS_ip_list[node],
port1=default_sw_port)
index = i+1
+ #for i in range(1, int(num_sw)+1):
+ #main.Mininet1.assign_sw_controller(
+ #sw=str(i),
+ #ip1="10.128.174.1",
+ # port1="6633")
main.log.info("Please check ptpd configuration to ensure "+\
"all nodes' system times are in sync")
@@ -246,43 +283,152 @@
MN1_ip+" --dport "+default_sw_port+" -j DROP")
main.log.info("Please wait for switch connection to timeout")
- time.sleep(60)
- if cluster_count >= 3:
- time.sleep(60)
- if cluster_count >= 5:
- time.sleep(30)
- if cluster_count >= 6:
- time.sleep(30)
+
+
+ #time.sleep(60)
+ #if cluster_count >= 3:
+ # time.sleep(60)
+ #if cluster_count >= 5:
+ # time.sleep(30)
+ #if cluster_count >= 6:
+ # time.sleep(30)
if cluster_count >= 3:
- main.ONOS1.tshark_grep("SYN, ACK",
- "/tmp/syn_ack_onos1_iter"+str(i)+".txt")
- main.ONOS2.tshark_grep("SYN, ACK",
- "/tmp/syn_ack_onos2_iter"+str(i)+".txt")
- main.ONOS3.tshark_grep("SYN, ACK",
- "/tmp/syn_ack_onos3_iter"+str(i)+".txt")
+ main.ONOS1.handle.sendline("tshark -i eth0 -t e | "+
+ "grep 'SYN, ACK' | grep '6633' >"+
+ "/tmp/syn_ack_onos1_iter"+str(i)+".txt &")
+ main.ONOS2.handle.sendline("tshark -i eth0 -t e | "+
+ "grep 'SYN, ACK' | grep '6633' >"+
+ "/tmp/syn_ack_onos2_iter"+str(i)+".txt &")
+ main.ONOS3.handle.sendline("tshark -i eth0 -t e | "+
+ "grep 'SYN, ACK' | grep '6633' >"+
+ "/tmp/syn_ack_onos3_iter"+str(i)+".txt &")
if cluster_count >= 4:
- main.ONOS4.tshark_grep("SYN, ACK",
- "/tmp/syn_ack_onos4_iter"+str(i)+".txt")
+ main.ONOS4.handle.sendline("tshark -i eth0 -t e | "+
+ "grep 'SYN, ACK' | grep '6633' >"+
+ "/tmp/syn_ack_onos4_iter"+str(i)+".txt &")
if cluster_count >= 5:
- main.ONOS5.tshark_grep("SYN, ACK",
- "/tmp/syn_ack_onos5_iter"+str(i)+".txt")
+ main.ONOS5.handle.sendline("tshark -i eth0 -t e | "+
+ "grep 'SYN, ACK' | grep '6633' >"+
+ "/tmp/syn_ack_onos5_iter"+str(i)+".txt &")
if cluster_count >= 6:
- main.ONOS6.tshark_grep("SYN, ACK",
- "/tmp/syn_ack_onos6_iter"+str(i)+".txt")
+ main.ONOS6.handle.sendline("tshark -i eth0 -t e | "+
+ "grep 'SYN, ACK' | grep '6633' >"+
+ "/tmp/syn_ack_onos6_iter"+str(i)+".txt &")
if cluster_count == 7:
- main.ONOS7.tshark_grep("SYN, ACK",
- "/tmp/syn_ack_onos7_iter"+str(i)+".txt")
+ main.ONOS7.handle.sendline("tshark -i eth0 -t e | "+
+ "grep 'SYN, ACK' | grep '6633' >"+
+ "/tmp/syn_ack_onos6_iter"+str(i)+".txt &")
+
+ #NOTE:
+ # Delay before checking devices to
+ # help prevent timing out from CLI
+ # due to multiple command drop
+ time.sleep(20)
- main.step("Flushing iptables and obtaining t0")
- t0_system = time.time()*1000
- main.ONOS1.handle.sendline("sudo iptables -F")
- main.ONOS2.handle.sendline("sudo iptables -F")
- main.ONOS3.handle.sendline("sudo iptables -F")
- main.ONOS4.handle.sendline("sudo iptables -F")
- main.ONOS5.handle.sendline("sudo iptables -F")
- main.ONOS6.handle.sendline("sudo iptables -F")
- main.ONOS7.handle.sendline("sudo iptables -F")
+ loop = True
+ loop_count = 0
+ device_count = 0
+ while loop_count < 60 and loop:
+ main.log.info("Checking devices for device down")
+
+ temp_len = 0
+ device_str1 = main.ONOS1cli.devices(
+ node_ip=ONOS_ip_list[1])
+ device_json1 = json.loads(device_str1)
+ json_len = len(device_json1)
+
+ for device1 in device_json1:
+ temp_len = temp_len + 1
+ if device1['available'] == True:
+ loop = True
+ break
+ #if I'm on the last json object and I still haven't
+ #broken out of the loop, it means there were
+ #no available devices
+ elif temp_len == json_len-1:
+ main.log.info("Temp length: "+str(temp_len))
+ main.step("Flushing iptables and obtaining t0")
+ t0_system = time.time()*1000
+
+ main.ONOS1.handle.sendline("sudo iptables -F")
+ main.ONOS2.handle.sendline("sudo iptables -F")
+ main.ONOS3.handle.sendline("sudo iptables -F")
+ main.ONOS4.handle.sendline("sudo iptables -F")
+ main.ONOS5.handle.sendline("sudo iptables -F")
+ main.ONOS6.handle.sendline("sudo iptables -F")
+ main.ONOS7.handle.sendline("sudo iptables -F")
+
+ loop = False
+ break
+
+ #if cluster_count == 1:
+ # device_str1 = main.ONOS1cli.devices(
+ # node_ip=ONOS_ip_list[1])
+ # device_json1 = json.loads(device_str1)
+ # for device1 in device_json1:
+ # if device1['available'] == False:
+ # device_count += 1
+ # else:
+ # device_count = 0
+ #if cluster_count == 2:
+ # device_str2 = main.ONOS2cli.devices(
+ # node_ip=ONOS_ip_list[2])
+ # device_json2 = json.loads(device_str2)
+ # for device2 in device_json2:
+ # if device2['available'] == False:
+ # device_count += 1
+ # else:
+ # device_count = 0
+ #if cluster_count == 3:
+ # device_str3 = main.ONOS3cli.devices(
+ # node_ip=ONOS_ip_list[3])
+ # device_json3 = json.loads(device_str3)
+ # for device3 in device_json3:
+ # if device3['available'] == False:
+ # device_count += 1
+ # else:
+ # device_count = 0
+ #if cluster_count == 4:
+ # device_str4 = main.ONOS4cli.devices(
+ # node_ip=ONOS_ip_list[4])
+ # device_json4 = json.loads(device_str4)
+ # for device4 in device_json4:
+ # if device4['available'] == False:
+ # device_count += 1
+ # else:
+ # device_count = 0
+ #if cluster_count == 5:
+ # device_str5 = main.ONOS5cli.devices(
+ # node_ip=ONOS_ip_list[5])
+ # device_json5 = json.loads(device_str5)
+ # for device5 in device_json5:
+ # if device5['available'] == False:
+ # device_count += 1
+ # else:
+ # device_count = 0
+ #if cluster_count == 7:
+ # device_str7 = main.ONOS7cli.devices(
+ # node_ip=ONOS_ip_list[7])
+ # device_json7 = json.loads(device_str7)
+ # for device7 in device_json7:
+ # if device7['available'] == False:
+ # device_count += 1
+ # else:
+ # device_count = 0
+
+ #If device count is greater than the
+ #number of switches discovered by all nodes
+ #then remove iptables and measure t0 system time
+
+
+ loop_count += 1
+ time.sleep(1)
+
+ if device_count < int(num_sw):
+ main.log.info("Devices down did not ")
+
+ main.log.info("System time t0: "+str(t0_system))
counter_loop = 0
counter_avail1 = 0
@@ -308,7 +454,8 @@
if node == 1 and not onos1_dev:
main.log.info("Checking node 1 for device "+
"discovery")
- device_str_obj1 = main.ONOS1cli.devices()
+ device_str_obj1 = main.ONOS1cli.devices(
+ node_ip=ONOS_ip_list[1])
device_json1 = json.loads(device_str_obj1)
for device1 in device_json1:
if device1['available'] == True:
@@ -322,7 +469,8 @@
if node == 2 and not onos2_dev:
main.log.info("Checking node 2 for device "+
"discovery")
- device_str_obj2 = main.ONOS2cli.devices()
+ device_str_obj2 = main.ONOS2cli.devices(
+ node_ip=ONOS_ip_list[2])
device_json2 = json.loads(device_str_obj2)
for device2 in device_json2:
if device2['available'] == True:
@@ -336,7 +484,8 @@
if node == 3 and not onos3_dev:
main.log.info("Checking node 3 for device "+
"discovery")
- device_str_obj3 = main.ONOS3cli.devices()
+ device_str_obj3 = main.ONOS3cli.devices(
+ node_ip=ONOS_ip_list[3])
device_json3 = json.loads(device_str_obj3)
for device3 in device_json3:
if device3['available'] == True:
@@ -350,7 +499,8 @@
if node == 4 and not onos4_dev:
main.log.info("Checking node 4 for device "+
"discovery")
- device_str_obj4 = main.ONOS4cli.devices()
+ device_str_obj4 = main.ONOS4cli.devices(
+ node_ip=ONOS_ip_list[4])
device_json4 = json.loads(device_str_obj4)
for device4 in device_json4:
if device4['available'] == True:
@@ -364,7 +514,8 @@
if node == 5 and not onos5_dev:
main.log.info("Checking node 5 for device "+
"discovery")
- device_str_obj5 = main.ONOS5cli.devices()
+ device_str_obj5 = main.ONOS5cli.devices(
+ node_ip=ONOS_ip_list[5])
device_json5 = json.loads(device_str_obj5)
for device5 in device_json5:
if device5['available'] == True:
@@ -378,7 +529,8 @@
if node == 6 and not onos6_dev:
main.log.info("Checking node 6 for device "+
"discovery")
- device_str_obj6 = main.ONOS6cli.devices()
+ device_str_obj6 = main.ONOS6cli.devices(
+ node_ip=ONOS_ip_list[6])
device_json6 = json.loads(device_str_obj6)
for device6 in device_json6:
if device6['available'] == True:
@@ -392,7 +544,8 @@
if node == 7 and not onos7_dev:
main.log.info("Checking node 7 for device "+
"discovery")
- device_str_obj7 = main.ONOS7cli.devices()
+ device_str_obj7 = main.ONOS7cli.devices(
+ node_ip=ONOS_ip_list[7])
device_json7 = json.loads(device_str_obj7)
for device7 in device_json7:
if device7['available'] == True:
@@ -413,6 +566,7 @@
if onos1_dev:
main.log.info("All devices have been discovered"+
" on all ONOS instances")
+ time.sleep(5)
json_str_metrics_1 =\
main.ONOS1cli.topology_events_metrics()
json_obj_1 = json.loads(json_str_metrics_1)
@@ -421,9 +575,13 @@
graph_lat_1 = \
int(graph_timestamp_1) - int(t0_system)
-
+
+ main.log.info("Graph Timestamp ONOS1: "+
+ str(graph_timestamp_1))
+
if graph_lat_1 > sw_disc_threshold_min\
- and graph_lat_1 < sw_disc_threshold_max:
+ and graph_lat_1 < sw_disc_threshold_max\
+ and int(i) > iter_ignore:
sw_discovery_lat_list.append(
graph_lat_1)
main.log.info("Sw discovery latency of "+
@@ -440,6 +598,8 @@
main.log.info("All devices have been discovered"+
" on all "+str(cluster_count)+
" ONOS instances")
+ time.sleep(5)
+
json_str_metrics_1 =\
main.ONOS1cli.topology_events_metrics()
json_str_metrics_2 =\
@@ -455,22 +615,27 @@
int(graph_timestamp_1) - int(t0_system)
graph_lat_2 = \
int(graph_timestamp_2) - int(t0_system)
+
+ main.log.info("Graph Timestamp ONOS1: "+
+ str(graph_timestamp_1))
+ main.log.info("Graph Timestamp ONOS2: "+
+ str(graph_timestamp_2))
- avg_graph_lat = \
- (int(graph_lat_1) +\
- int(graph_lat_2)) / 2
+ max_graph_lat = max(graph_lat_1,
+ graph_lat_2, graph_lat_3)
- if avg_graph_lat > sw_disc_threshold_min\
- and avg_graph_lat < sw_disc_threshold_max:
+ if max_graph_lat > sw_disc_threshold_min\
+ and max_graph_lat < sw_disc_threshold_max\
+ and int(i) > iter_ignore:
sw_discovery_lat_list.append(
- avg_graph_lat)
+ max_graph_lat)
main.log.info("Sw discovery latency of "+
str(cluster_count)+" node(s): "+
- str(avg_graph_lat)+" ms")
+ str(max_graph_lat)+" ms")
else:
main.log.info("Switch discovery latency "+
"exceeded the threshold.")
- main.log.info(avg_graph_lat)
+ main.log.info(max_graph_lat)
break
if cluster_count == 3:
if onos1_dev and onos2_dev and onos3_dev:
@@ -481,7 +646,7 @@
#TODO: Investigate this sleep
# added to 'pad' the results with
# plenty of time to 'catch up'
- time.sleep(20)
+ time.sleep(5)
json_str_metrics_1 =\
main.ONOS1cli.topology_events_metrics()
@@ -506,22 +671,29 @@
graph_lat_3 = \
int(graph_timestamp_3) - int(t0_system)
- avg_graph_lat = \
- (int(graph_lat_1) +\
- int(graph_lat_2) +\
- int(graph_lat_3)) / 3
-
- if avg_graph_lat > sw_disc_threshold_min\
- and avg_graph_lat < sw_disc_threshold_max:
+ main.log.info("Graph Timestamp ONOS1: "+
+ str(graph_timestamp_1))
+ main.log.info("Graph Timestamp ONOS2: "+
+ str(graph_timestamp_2))
+ main.log.info("Graph Timestamp ONOS3: "+
+ str(graph_timestamp_3))
+
+ max_graph_lat = max(graph_lat_1,
+ graph_lat_2,
+ graph_lat_3)
+
+ if max_graph_lat > sw_disc_threshold_min\
+ and max_graph_lat < sw_disc_threshold_max\
+ and int(i) > iter_ignore:
sw_discovery_lat_list.append(
- avg_graph_lat)
+ max_graph_lat)
main.log.info("Sw discovery latency of "+
str(cluster_count)+" node(s): "+
- str(avg_graph_lat)+" ms")
+ str(max_graph_lat)+" ms")
else:
main.log.info("Switch discovery latency "+
"exceeded the threshold.")
- main.log.info(avg_graph_lat)
+ main.log.info(max_graph_lat)
break
if cluster_count == 4:
@@ -558,24 +730,33 @@
int(graph_timestamp_3) - int(t0_system)
graph_lat_4 = \
int(graph_timestamp_4) - int(t0_system)
-
- avg_graph_lat = \
- (int(graph_lat_1) +\
- int(graph_lat_2) +\
- int(graph_lat_3) +\
- int(graph_lat_4)) / 4
- if avg_graph_lat > sw_disc_threshold_min\
- and avg_graph_lat < sw_disc_threshold_max:
+ main.log.info("Graph Timestamp ONOS1: "+
+ str(graph_timestamp_1))
+ main.log.info("Graph Timestamp ONOS2: "+
+ str(graph_timestamp_2))
+ main.log.info("Graph Timestamp ONOS3: "+
+ str(graph_timestamp_3))
+ main.log.info("Graph Timestamp ONOS4: "+
+ str(graph_timestamp_4))
+
+ max_graph_lat = max(graph_lat_1,
+ graph_lat_2,
+ graph_lat_3,
+ graph_lat_4)
+
+ if max_graph_lat > sw_disc_threshold_min\
+ and max_graph_lat < sw_disc_threshold_max\
+ and int(i) > iter_ignore:
sw_discovery_lat_list.append(
- avg_graph_lat)
+ max_graph_lat)
main.log.info("Sw discovery latency of "+
str(cluster_count)+" node(s): "+
- str(avg_graph_lat)+" ms")
+ str(max_graph_lat)+" ms")
else:
main.log.info("Switch discovery latency "+
"exceeded the threshold.")
- main.log.info(avg_graph_lat)
+ main.log.info(max_graph_lat)
break
if cluster_count == 5:
@@ -587,7 +768,7 @@
#TODO: Investigate this sleep
# added to 'pad' the results with
# plenty of time to 'catch up'
- time.sleep(20)
+ time.sleep(5)
json_str_metrics_1 =\
main.ONOS1cli.topology_events_metrics()
@@ -625,25 +806,36 @@
int(graph_timestamp_4) - int(t0_system)
graph_lat_5 = \
int(graph_timestamp_5) - int(t0_system)
-
- avg_graph_lat = \
- (int(graph_lat_1) +\
- int(graph_lat_2) +\
- int(graph_lat_3) +\
- int(graph_lat_4) +\
- int(graph_lat_5)) / 5
- if avg_graph_lat > sw_disc_threshold_min\
- and avg_graph_lat < sw_disc_threshold_max:
+ main.log.info("Graph Timestamp ONOS1: "+
+ str(graph_timestamp_1))
+ main.log.info("Graph Timestamp ONOS2: "+
+ str(graph_timestamp_2))
+ main.log.info("Graph Timestamp ONOS3: "+
+ str(graph_timestamp_3))
+ main.log.info("Graph Timestamp ONOS4: "+
+ str(graph_timestamp_4))
+ main.log.info("Graph Timestamp ONOS5: "+
+ str(graph_timestamp_5))
+
+ max_graph_lat = max(graph_lat_1,
+ graph_lat_2,
+ graph_lat_3,
+ graph_lat_4,
+ graph_lat_5)
+
+ if max_graph_lat > sw_disc_threshold_min\
+ and max_graph_lat < sw_disc_threshold_max\
+ and int(i) > iter_ignore:
sw_discovery_lat_list.append(
- avg_graph_lat)
+ max_graph_lat)
main.log.info("Sw discovery latency of "+
str(cluster_count)+" node(s): "+
- str(avg_graph_lat)+" ms")
+ str(max_graph_lat)+" ms")
else:
main.log.info("Switch discovery latency "+
"exceeded the threshold.")
- main.log.info(avg_graph_lat)
+ main.log.info(max_graph_lat)
break
if cluster_count == 6:
@@ -694,26 +886,39 @@
int(graph_timestamp_5) - int(t0_system)
graph_lat_6 = \
int(graph_timestamp_6) - int(t0_system)
-
- avg_graph_lat = \
- (int(graph_lat_1) +\
- int(graph_lat_2) +\
- int(graph_lat_3) +\
- int(graph_lat_4) +\
- int(graph_lat_5) +\
- int(graph_lat_6)) / 6
- if avg_graph_lat > sw_disc_threshold_min\
- and avg_graph_lat < sw_disc_threshold_max:
+ main.log.info("Graph Timestamp ONOS1: "+
+ str(graph_timestamp_1))
+ main.log.info("Graph Timestamp ONOS2: "+
+ str(graph_timestamp_2))
+ main.log.info("Graph Timestamp ONOS3: "+
+ str(graph_timestamp_3))
+ main.log.info("Graph Timestamp ONOS4: "+
+ str(graph_timestamp_4))
+ main.log.info("Graph Timestamp ONOS5: "+
+ str(graph_timestamp_5))
+ main.log.info("Graph Timestamp ONOS6: "+
+ str(graph_timestamp_6))
+
+ max_graph_lat = max(graph_lat_1,
+ graph_lat_2,
+ graph_lat_3,
+ graph_lat_4,
+ graph_lat_5,
+ graph_lat_6)
+
+ if max_graph_lat > sw_disc_threshold_min\
+ and max_graph_lat < sw_disc_threshold_max\
+ and int(i) > iter_ignore:
sw_discovery_lat_list.append(
- avg_graph_lat)
+ max_graph_lat)
main.log.info("Sw discovery latency of "+
str(cluster_count)+" node(s): "+
- str(avg_graph_lat)+" ms")
+ str(max_graph_lat)+" ms")
else:
main.log.info("Switch discovery latency "+
"exceeded the threshold.")
- main.log.info(avg_graph_lat)
+ main.log.info(max_graph_lat)
break
if cluster_count == 7:
@@ -726,7 +931,7 @@
#TODO: Investigate this sleep
# added to 'pad' the results with
# plenty of time to 'catch up'
- time.sleep(20)
+ time.sleep(5)
json_str_metrics_1 =\
main.ONOS1cli.topology_events_metrics()
@@ -779,35 +984,55 @@
graph_lat_7 = \
int(graph_timestamp_7) - int(t0_system)
- avg_graph_lat = \
- (int(graph_lat_1) +\
- int(graph_lat_2) +\
- int(graph_lat_3) +\
- int(graph_lat_4) +\
- int(graph_lat_5) +\
- int(graph_lat_6) +\
- int(graph_lat_7)) / 7
+ main.log.info("Graph Timestamp ONOS1: "+
+ str(graph_timestamp_1))
+ main.log.info("Graph Timestamp ONOS2: "+
+ str(graph_timestamp_2))
+ main.log.info("Graph Timestamp ONOS3: "+
+ str(graph_timestamp_3))
+ main.log.info("Graph Timestamp ONOS4: "+
+ str(graph_timestamp_4))
+ main.log.info("Graph Timestamp ONOS5: "+
+ str(graph_timestamp_5))
+ main.log.info("Graph Timestamp ONOS6: "+
+ str(graph_timestamp_6))
+ main.log.info("Graph Timestamp ONOS7: "+
+ str(graph_timestamp_7))
+
+ max_graph_lat = max(graph_lat_1,
+ graph_lat_2,
+ graph_lat_3,
+ graph_lat_4,
+ graph_lat_5,
+ graph_lat_6,
+ graph_lat_7)
- if avg_graph_lat > sw_disc_threshold_min\
- and avg_graph_lat < sw_disc_threshold_max:
+ if max_graph_lat > sw_disc_threshold_min\
+ and max_graph_lat < sw_disc_threshold_max\
+ and int(i) > iter_ignore:
sw_discovery_lat_list.append(
- avg_graph_lat)
+ max_graph_lat)
main.log.info("Sw discovery latency of "+
str(cluster_count)+" node(s): "+
- str(avg_graph_lat)+" ms")
+ str(max_graph_lat)+" ms")
else:
main.log.info("Switch discovery latency "+
"exceeded the threshold.")
- main.log.info(avg_graph_lat)
+ main.log.info(max_graph_lat)
break
counter_loop += 1
time.sleep(3)
#END WHILE LOOP
-
+
+ #Below is used for reporting SYN / ACK timing
+ #of all switches
main.ONOS1.tshark_stop()
syn_ack_timestamp_list = []
+ if cluster_count < 3:
+ #TODO: capture synack on nodes less than 3
+ syn_ack_timestamp_list.append(0)
if cluster_count >= 3:
main.ONOS2.tshark_stop()
@@ -826,17 +1051,29 @@
f_onos1:
for line in f_onos1:
line = line.split(" ")
- syn_ack_timestamp_list.append(line[1])
+ try:
+ float(line[1])
+ syn_ack_timestamp_list.append(line[1])
+ except ValueError:
+ main.log.info("String cannot be converted")
with open("/tmp/syn_ack_onos2_iter"+str(i)+".txt") as\
f_onos2:
for line in f_onos2:
line = line.split(" ")
- syn_ack_timestamp_list.append(line[1])
+ try:
+ float(line[1])
+ syn_ack_timestamp_list.append(line[1])
+ except ValueError:
+ main.log.info("String cannot be converted")
with open("/tmp/syn_ack_onos3_iter"+str(i)+".txt") as\
f_onos3:
for line in f_onos3:
line = line.split(" ")
- syn_ack_timestamp_list.append(line[1])
+ try:
+ float(line[1])
+ syn_ack_timestamp_list.append(line[1])
+ except ValueError:
+ main.log.info("String cannot be converted")
if cluster_count >= 4:
main.ONOS4.tshark_stop()
time.sleep(5)
@@ -847,7 +1084,11 @@
f_onos4:
for line in f_onos4:
line = line.split(" ")
- syn_ack_timestamp_list.append(line[1])
+ try:
+ float(line[1])
+ syn_ack_timestamp_list.append(line[1])
+ except ValueError:
+ main.log.info("String cannot be converted")
if cluster_count >= 5:
main.ONOS5.tshark_stop()
time.sleep(5)
@@ -858,7 +1099,11 @@
f_onos5:
for line in f_onos5:
line = line.split(" ")
- syn_ack_timestamp_list.append(line[1])
+ try:
+ float(line[1])
+ syn_ack_timestamp_list.append(line[1])
+ except ValueError:
+ main.log.info("String cannot be converted")
if cluster_count >= 6:
main.ONOS6.tshark_stop()
time.sleep(5)
@@ -869,7 +1114,11 @@
f_onos6:
for line in f_onos6:
line = line.split(" ")
- syn_ack_timestamp_list.append(line[1])
+ try:
+ float(line[1])
+ syn_ack_timestamp_list.append(line[1])
+ except ValueError:
+ main.log.info("String cannot be converted")
if cluster_count == 7:
main.ONOS7.tshark_stop()
time.sleep(5)
@@ -880,11 +1129,16 @@
f_onos7:
for line in f_onos7:
line = line.split(" ")
- syn_ack_timestamp_list.append(line[1])
+ try:
+ float(line[1])
+ syn_ack_timestamp_list.append(line[1])
+ except ValueError:
+ main.log.info("String cannot be converted")
#Sort the list by timestamp
syn_ack_timestamp_list = sorted(syn_ack_timestamp_list)
-
+ print "syn_ack_-1 " + str(syn_ack_timestamp_list)
+
syn_ack_delta =\
int(float(syn_ack_timestamp_list[-1])*1000) -\
int(float(syn_ack_timestamp_list[0])*1000)
@@ -906,11 +1160,10 @@
"(last sw SYN/ACK time - first sw SYN/ACK time) "+
str(sum(syn_ack_delta_list)/len(syn_ack_delta_list)) +
" ms")
- main.log.report("Switch discovery lat for "+\
- str(cluster_count)+" instance(s), 100 sw each: ")
- main.log.report("Avg: "+str(sw_lat_avg)+" ms")
- main.log.report("Std Deviation: "+
- str(round(sw_lat_dev,1))+" ms")
+ main.log.report(str(num_sw)+" Switch discovery lat for "+\
+ str(cluster_count)+" instance(s): ")
+ main.log.report("Avg: "+str(sw_lat_avg)+" ms "+
+ "Std Deviation: "+str(round(sw_lat_dev,1))+" ms")
utilities.assert_equals(expect=main.TRUE, actual=assertion,
onpass="Switch discovery convergence latency" +\
@@ -946,42 +1199,59 @@
#cluster count and start from 3.
#You can optionally change the increment to
#test steps of node sizes, such as 3,5,7
-
+
global cluster_count
cluster_count += 2
-
+ main.log.report("Increasing cluster size to "+
+ str(cluster_count))
+
install_result = main.FALSE
#Supports up to 7 node configuration
#TODO: Cleanup this ridiculous repetitive code
+ if cluster_count == 3:
+ install_result = \
+ main.ONOSbench.onos_install(node=ONOS2_ip)
+ install_result = \
+ main.ONOSbench.onos_install(node=ONOS3_ip)
+ time.sleep(5)
+ main.log.info("Starting CLI")
+ main.ONOS2cli.start_onos_cli(ONOS2_ip)
+ main.ONOS3cli.start_onos_cli(ONOS3_ip)
+ main.ONOS1cli.add_node(ONOS2_ip, ONOS2_ip)
+ main.ONOS1cli.add_node(ONOS3_ip, ONOS3_ip)
+
if cluster_count == 4:
main.log.info("Installing ONOS on node 4")
install_result = \
main.ONOSbench.onos_install(node=ONOS4_ip)
+ time.sleep(5)
main.log.info("Starting CLI")
main.ONOS4cli.start_onos_cli(ONOS4_ip)
main.ONOS1cli.add_node(ONOS4_ip, ONOS4_ip)
elif cluster_count == 5:
- main.log.info("Installing ONOS on node 5")
- install_result1 = \
- main.ONOSbench.onos_install(options="",node=ONOS4_ip)
+ main.log.info("Installing ONOS on nodes 4 and 5")
install_result2 = \
- main.ONOSbench.onos_install(node=ONOS5_ip)
+ main.ONOSbench.onos_install(options="",node=ONOS4_ip)
+ install_result3 = \
+ main.ONOSbench.onos_install(options="",node=ONOS5_ip)
+ time.sleep(5)
main.log.info("Starting CLI")
main.ONOS4cli.start_onos_cli(ONOS4_ip)
main.ONOS5cli.start_onos_cli(ONOS5_ip)
main.ONOS1cli.add_node(ONOS4_ip, ONOS4_ip)
main.ONOS1cli.add_node(ONOS5_ip, ONOS5_ip)
- install_result = install_result1 and install_result2
+ install_result = install_result2 and install_result3
elif cluster_count == 6:
- main.log.info("Installing ONOS on node 6")
+ main.log.info("Installing ONOS on nodes 4, 5,and 6")
install_result1 = \
main.ONOSbench.onos_install(options="",node=ONOS4_ip)
install_result2 = \
main.ONOSbench.onos_install(options="",node=ONOS5_ip)
install_result3 = \
main.ONOSbench.onos_install(node=ONOS6_ip)
+ time.sleep(5)
main.log.info("Starting CLI")
main.ONOS4cli.start_onos_cli(ONOS4_ip)
main.ONOS5cli.start_onos_cli(ONOS5_ip)
@@ -993,13 +1263,9 @@
install_result3
elif cluster_count == 7:
- main.log.info("Installing ONOS on node 7")
- install_result1 = \
- main.ONOSbench.onos_install(options="",node=ONOS4_ip)
- install_result2 = \
- main.ONOSbench.onos_install(options="",node=ONOS5_ip)
+ main.log.info("Installing ONOS on nodes 4, 5, 6,and 7")
install_result3 = \
- main.ONOSbench.onos_install(options="",node=ONOS6_ip)
+ main.ONOSbench.onos_install(node=ONOS6_ip)
install_result4 = \
main.ONOSbench.onos_install(node=ONOS7_ip)
main.log.info("Starting CLI")
@@ -1012,7 +1278,7 @@
main.ONOS1cli.add_node(ONOS6_ip, ONOS6_ip)
main.ONOS1cli.add_node(ONOS7_ip, ONOS7_ip)
- install_result = install_result1 and install_result2 and\
+ install_result = \
install_result3 and install_result4
time.sleep(5)
@@ -1027,3 +1293,39 @@
" nodes successful",
onfail="Scale out to "+str(cluster_count)+\
" nodes failed")
+
+
+ def CASE4(self, main):
+ '''
+ Cleanup ONOS nodes and Increase topology size
+ '''
+ #TODO: use meaningful assertion
+ assertion=main.TRUE
+
+ ONOS1_ip = main.params['CTRL']['ip1']
+ ONOS2_ip = main.params['CTRL']['ip2']
+ ONOS3_ip = main.params['CTRL']['ip3']
+ ONOS4_ip = main.params['CTRL']['ip4']
+ ONOS5_ip = main.params['CTRL']['ip5']
+ ONOS6_ip = main.params['CTRL']['ip6']
+ ONOS7_ip = main.params['CTRL']['ip7']
+ MN1_ip = main.params['MN']['ip1']
+ BENCH_ip = main.params['BENCH']['ip']
+
+ main.log.info("Uninstalling previous instances")
+ main.ONOSbench.onos_uninstall(node_ip = ONOS2_ip)
+ main.ONOSbench.onos_uninstall(node_ip = ONOS3_ip)
+ main.ONOSbench.onos_uninstall(node_ip = ONOS4_ip)
+ main.ONOSbench.onos_uninstall(node_ip = ONOS5_ip)
+ main.ONOSbench.onos_uninstall(node_ip = ONOS6_ip)
+ main.ONOSbench.onos_uninstall(node_ip = ONOS7_ip)
+
+ global topo_iteration
+ global cluster_count
+ cluster_count = 1
+ topo_iteration += 1
+
+ main.log.report("Increasing topology size")
+ utilities.assert_equals(expect=main.TRUE, actual=assertion,
+ onpass="Topology size increased successfully",
+ onfail="Topology size was not increased")
diff --git a/TestON/tests/TopoConvNext/TopoConvNext.topo b/TestON/tests/TopoConvNext/TopoConvNext.topo
index 855e688..b7e9e96 100644
--- a/TestON/tests/TopoConvNext/TopoConvNext.topo
+++ b/TestON/tests/TopoConvNext/TopoConvNext.topo
@@ -143,7 +143,7 @@
<type>MininetCliDriver</type>
<connect_order>16</connect_order>
<COMPONENTS>
- <arg1> --custom topo-700sw.py </arg1>
+ <arg1> --custom topo-500sw.py </arg1>
<arg2> --arp --mac --topo mytopo</arg2>
<arg3> </arg3>
<controller> remote </controller>
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.params b/TestON/tests/TopoPerfNext/TopoPerfNext.params
index 7eac6ef..42512e8 100644
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.params
+++ b/TestON/tests/TopoPerfNext/TopoPerfNext.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,3,4</testcases>
+ <testcases>1,2,3</testcases>
<ENV>
<cellName>topo_perf_test</cellName>
@@ -38,6 +38,13 @@
<debugMode>off</debugMode>
<onosLogFile>/opt/onos/log/karaf*</onosLogFile>
+ <topo_config_file>
+ single_topo_event_accumulator.cfg
+ </topo_config_file>
+ <topo_config_name>
+ org.onlab.onos.net.topology.impl.DefaultTopologyProvider.cfg
+ </topo_config_name>
+
#Number of times to iterate each case
<numIter>20</numIter>
<numSwitch>2</numSwitch>
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.py b/TestON/tests/TopoPerfNext/TopoPerfNext.py
index a710f82..49089bf 100644
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.py
+++ b/TestON/tests/TopoPerfNext/TopoPerfNext.py
@@ -42,7 +42,18 @@
MN1_ip = main.params['MN']['ip1']
BENCH_ip = main.params['BENCH']['ip']
+ topo_cfg_file = main.params['TEST']['topo_config_file']
+ topo_cfg_name = main.params['TEST']['topo_config_name']
+
main.case("Setting up test environment")
+ main.log.info("Copying topology event accumulator config"+\
+ " to ONOS /ppackage/etc")
+ main.ONOSbench.handle.sendline("cp ~/"+\
+ topo_cfg_file+\
+ "~/ONOS/tools/package/etc/"+\
+ topo_cfg_name)
+ main.ONOSbench.handle.expect("\$")
+
main.log.report("Setting up test environment")
main.step("Cleaning previously installed ONOS if any")
@@ -75,6 +86,10 @@
pull_result = main.TRUE
main.log.info("Skipped git checkout and pull")
+ #TODO: Uncomment when wiki posting works
+ #main.log.report("Commit information - ")
+ #main.ONOSbench.get_version(report=True)
+
main.step("Using mvn clean & install")
#mvn_result = main.ONOSbench.clean_install()
mvn_result = main.TRUE
@@ -360,6 +375,17 @@
and avg_delta_ofp_graph < threshold_max\
and int(i) > iter_ignore:
latency_ofp_to_graph_list.append(avg_delta_ofp_graph)
+ elif avg_delta_ofp_graph > (-10) and \
+ avg_delta_ofp_graph < 0.0 and\
+ int(i) > iter_ignore:
+ main.log.info("Sub-millisecond result likely; "+
+ "negative result was rounded to 0")
+ #NOTE: Current metrics framework does not
+ #support sub-millisecond accuracy. Therefore,
+ #if the result is negative, we can reasonably
+ #conclude sub-millisecond results and just
+ #append the best rounded effort - 0 ms.
+ latency_ofp_to_graph_list.append(0)
else:
main.log.info("Results for ofp-to-graph "+\
"ignored due to excess in threshold")
@@ -509,7 +535,9 @@
"Avg: "+str(latency_end_to_end_avg)+" ms "+
"Std Deviation: "+latency_end_to_end_std_dev+" ms")
main.log.report("Switch add - OFP-to-Graph latency: "+\
- "Avg: "+str(latency_ofp_to_graph_avg)+" ms "+
+ "Note: results are not accurate to sub-millisecond. "+
+ "Any sub-millisecond results are rounded to 0 ms. ")
+ main.log.report("Avg: "+str(latency_ofp_to_graph_avg)+" ms "+
"Std Deviation: "+latency_ofp_to_graph_std_dev+" ms")
main.log.report("Switch add - TCP-to-OFP latency: "+\
"Avg: "+str(latency_tcp_to_ofp_avg)+" ms "+