Updated drivers and tests from HA. Changed include:
- onos-check-logs function
- remove some print statements
- create a csv file for jenkin's plugin
- Add timestamp to long pings
- Remove cookies from flow table comparison
- when printing flow tables, check that a correct protocol format is
given
- add cli commands for leadership election sample app
- add command to check that all devices have a masster assigned
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 3a24008..80fe0de 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -110,7 +110,7 @@
main.LogFileName = main.logdir + "/" + main.TEST + "_" +str(currentTime) + ".log"
main.ReportFileName = main.logdir + "/" + main.TEST + "_" + str(currentTime) + ".rpt"
- main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
+ main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
#### Add log-level - Report
logging.addLevelName(9, "REPORT")
@@ -229,14 +229,14 @@
#main.log.report(testResult)
main.testResult = testResult
main.log.exact(testResult)
-
+
##CSV output needed for Jenkin's plot plugin
#NOTE: the elements were orded based on the colors assigned to the data
logfile = open(main.JenkinsCSV ,"w")
logfile.write(",".join( ['Tests Failed', 'Tests Passed', 'Tests Planned'] ) + "\n")
- logfile.write(",".join( [str(main.TOTAL_TC_FAIL), str(main.TOTAL_TC_PASS), str(main.TOTAL_TC_PLANNED)] ))
- logfile.close()
-
+ logfile.write(",".join( [str(int(main.TOTAL_TC_FAIL)), str(int(main.TOTAL_TC_PASS)), str(int(main.TOTAL_TC_PLANNED))] ))
+ logfile.close()
+
diff --git a/TestON/dependencies/rotate.sh b/TestON/dependencies/rotate.sh
index 235d138..c685a14 100755
--- a/TestON/dependencies/rotate.sh
+++ b/TestON/dependencies/rotate.sh
@@ -51,4 +51,4 @@
#Maybe this should be an argument? pack-and-rotate supports that
nr_max=20
-pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap" ${nr_max}
+pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap ${root_dir}/${base_name}*.log " ${nr_max}
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 17c25a0..bf3a2c6 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -1207,7 +1207,7 @@
break
mn_ports.sort(key=float)
onos_ports.sort(key=float)
- #print "\nPorts for Switch %s:" % (switch['name'])
+ #print "\nPorts for Switch %s:" % (mn_switch['name'])
#print "\tmn_ports[] = ", mn_ports
#print "\tonos_ports[] = ", onos_ports
mn_ports_log = mn_ports
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index 1795a16..ae0f599 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -114,7 +114,7 @@
args = utilities.parse_args(["SRC","TARGET","PINGTIME"],**pingParams)
precmd = "sudo rm /tmp/ping." + args["SRC"]
self.execute(cmd=precmd,prompt="(.*)",timeout=10)
- command = "sudo mininet/util/m " + args["SRC"] + " ping "+args ["TARGET"]+" -i .2 -w " + str(args['PINGTIME']) + " > /tmp/ping." + args["SRC"] + " &"
+ command = "sudo mininet/util/m " + args["SRC"] + " ping "+args ["TARGET"]+" -i .2 -w " + str(args['PINGTIME']) + " -D > /tmp/ping." + args["SRC"] + " &"
main.log.info( command )
self.execute(cmd=command,prompt="(.*)",timeout=10)
self.handle.sendline("")
@@ -372,57 +372,40 @@
def get_flowTable(self, protoVersion, sw):
#TODO document usage
- #FIXME: clean up print statements and such
+ #TODO add option to look at cookies. ignoreing them for now
self.handle.sendline("cd")
self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
- #TODO: Write seperate versions of the function for this, possibly a string that tells it which switch is in use?
- #For 1.0 version of OVS
- #command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $6 $7 }' |sort -n -k1"
- #for 1.3 version of OVS
- #command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $3 $7 $8}' |sort -n -k1"
+ #print "get_flowTable(" + str(protoVersion) +" " + str(sw) +")"
#NOTE: Use format to force consistent flow table output across versions
if protoVersion==1.0:
- command = "sudo ovs-ofctl dump-flows " + sw + " -F OpenFlow10-table_id | awk '{OFS=\",\" ; print $1 $3 $6 $7 $8}' |sort -n -k1"
+ command = "sudo ovs-ofctl dump-flows " + sw + " -F OpenFlow10-table_id | awk '{OFS=\",\" ; print $1 $3 $6 $7 $8}' | cut -d ',' -f 2- | sort -n -k1 -r"
self.handle.sendline(command)
- self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
+ self.handle.expect(["k1 -r",pexpect.EOF,pexpect.TIMEOUT])
self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
response = self.handle.before
#print "response=", response
return response
elif protoVersion==1.3:
- command = "sudo ovs-ofctl dump-flows " + sw + " -O OpenFlow13 | awk '{OFS=\",\" ; print $1 $3 $6 $7}' |sort -n -k1"
+ command = "sudo ovs-ofctl dump-flows " + sw + " -O OpenFlow13 | awk '{OFS=\",\" ; print $1 $3 $6 $7}' | cut -d ',' -f 2- | sort -n -k1 -r"
self.handle.sendline(command)
- #print "ovs-vsctl Command sent status."
- #self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
- #print "sort return status: "
- #print self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
- #print self.handle.before
+ self.handle.expect(["k1 -r",pexpect.EOF,pexpect.TIMEOUT])
self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
- #print "OFPST_FLOW expected status: "
- #print self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
- #print self.handle.before
response = self.handle.before
#print "response=", response
- return response
-
+ return response
+ else:
+ main.log.error("Unknown protoVersion in get_flowTable(). given: ("+str(type(protoVersion))+") '"+str(protoVersion)+"'")
+
def flow_comp(self,flow1,flow2):
- #print "Inside flow compare function"
- #print "Flow1 Table:"
- #for flow in flow1:
- #print flow1
- #print "Flow2 Table"
- #for flow in flow2:
- #print flow2
-
if flow1==flow2:
return main.TRUE
else:
main.log.info("Flow tables do not match, printing tables:")
- #main.log.info("Flow Table 1:")
- #main.log.info(flow1)
- #main.log.info("Flow Table 2:")
- #main.log.info(flow2)
+ main.log.info("Flow Table 1:")
+ main.log.info(flow1)
+ main.log.info("Flow Table 2:")
+ main.log.info(flow2)
return main.FALSE
def setIpTablesOUTPUT(self, dst_ip, dst_port, action='add', packet_type='tcp',rule='DROP'):
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index a6cf341..358945f 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -693,7 +693,38 @@
main.log.info(self.name+" ::::::")
main.cleanup()
main.exit()
-
+
+ def roles_not_null(self):
+ '''
+ Iterates through each device and checks if there is a master assigned
+ Returns: main.TRUE if each device has a master
+ main.FALSE any device has no master
+ '''
+ try:
+ import json
+ raw_roles = self.roles()
+ roles_json = json.loads(raw_roles)
+ #search json for the device with id then return the device
+ for device in roles_json:
+ #print device
+ if device['master'] == "none":
+ main.log.warn("Device has no master: " + str(device) )
+ return main.FALSE
+ return main.TRUE
+
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
+
+
def paths(self, src_id, dst_id):
'''
Returns string of paths, and the cost.
@@ -1713,7 +1744,7 @@
onos_node is the ip of one of the onos nodes in the cluster
role must be either master, standby, or none
- Returns main.TRUE or main.FALSE based argument varification.
+ Returns main.TRUE or main.FALSE based on argument verification.
When device-role supports errors this should be extended to
support that output
'''
@@ -1805,5 +1836,115 @@
main.cleanup()
main.exit()
+ def election_test_leader(self):
+ '''
+ * CLI command to get the current leader for the Election test application.
+ #NOTE: Requires installation of the onos-app-election feature
+ Returns: Node IP of the leader if one exists
+ None if none exists
+ Main.FALSE on error
+ '''
+ try:
+ self.handle.sendline("election-test-leader")
+ self.handle.expect("election-test-leader")
+ self.handle.expect("onos>")
+ response = self.handle.before
+ #Leader
+ node_search = re.search("The\scurrent\sleader\sfor\sthe\sElection\sapp\sis\s(?P<node>.+)\.", response)
+ if node_search:
+ node = node_search.group('node')
+ main.log.info("Election-test-leader found " + node + " as the leader")
+ return node
+ #no leader
+ null_search = re.search("There\sis\scurrently\sno\sleader\selected\sfor\sthe\sElection\sapp", response)
+ if null_search:
+ main.log.info("Election-test-leader found no leader")
+ return None
+ #error
+ main.log.error("Error in election_test_leader: unexpected response")
+ main.log.error( repr(response) )
+ return main.FALSE
+
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
+
+ def election_test_run(self):
+ '''
+ * CLI command to run for leadership of the Election test application.
+ #NOTE: Requires installation of the onos-app-election feature
+ Returns: Main.TRUE on success
+ Main.FALSE on error
+ '''
+ try:
+ self.handle.sendline("election-test-run")
+ self.handle.expect("election-test-run")
+ self.handle.expect("onos>")
+ response = self.handle.before
+ #success
+ search = re.search("Entering\sleadership\selections\sfor\sthe\sElection\sapp.", response)
+ if search:
+ main.log.info("Entering leadership elections for the Election app.")
+ return main.TRUE
+ #error
+ main.log.error("Error in election_test_run: unexpected response")
+ main.log.error( repr(response) )
+ return main.FALSE
+
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
+
+ def election_test_withdraw(self):
+ '''
+ * CLI command to withdraw the local node from leadership election for
+ * the Election test application.
+ #NOTE: Requires installation of the onos-app-election feature
+ Returns: Main.TRUE on success
+ Main.FALSE on error
+ '''
+ try:
+ self.handle.sendline("election-test-withdraw")
+ self.handle.expect("election-test-withdraw")
+ self.handle.expect("onos>")
+ response = self.handle.before
+ #success
+ search = re.search("Withdrawing\sfrom\sleadership\selections\sfor\sthe\sElection\sapp.", response)
+ if search:
+ main.log.info("Withdrawing from leadership elections for the Election app.")
+ return main.TRUE
+ #error
+ main.log.error("Error in election_test_withdraw: unexpected response")
+ main.log.error( repr(response) )
+ return main.FALSE
+
+
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
#***********************************
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 418554d..678bae4 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -1072,21 +1072,21 @@
if not line.startswith("time="):
continue
#else
- print line
+ #print line
for var in line.split(","):
#print "'"+var+"'"
#print "'"+var.strip()+"'"
key, value = var.strip().split("=")
topology[key] = value
- print "topology = ", topology
+ #print "topology = ", topology
devices = topology.get('devices', False)
- print "devices = ", devices
+ #print "devices = ", devices
links = topology.get('links', False)
- print "links = ", links
- clusters = topology.get('clusters', False)
- print "clusters = ", clusters
+ #print "links = ", links
+ SCCs = topology.get('SCC(s)', False)
+ #print "SCCs = ", SCCs
paths = topology.get('paths', False)
- print "paths = ", paths
+ #print "paths = ", paths
return topology
except pexpect.EOF:
@@ -1339,3 +1339,26 @@
main.log.error( traceback.print_exc())
main.log.info(self.name+" ::::::")
+ def check_logs(self, onos_ip):
+ '''
+ runs onos-check-logs on the given onos node
+ returns the response
+ '''
+ try:
+ cmd = "onos-check-logs " + str(onos_ip)
+ self.handle.sendline(cmd)
+ self.handle.expect(cmd)
+ self.handle.expect("\$")
+ response = self.handle.before
+ return response
+ except pexpect.EOF:
+ main.log.error("Lost ssh connection")
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ except:
+ main.log.error("Some error in check_logs:")
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+
+
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 00d00ed..878839e 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -48,7 +48,7 @@
ssh_newkey = 'Are you sure you want to continue connecting'
refused = "ssh: connect to host "+self.ip_address+" port 22: Connection refused"
if self.port:
- self.handle =pexpect.spawn('ssh -p '+self.port+' '+self.user_name+'@'+self.ip_address,maxread=50000)
+ self.handle =pexpect.spawn('ssh -p '+self.port+' '+self.user_name+'@'+self.ip_address, env = {"TERM": "vt100"} , maxread=50000)
else :
self.handle =pexpect.spawn('ssh -X '+self.user_name+'@'+self.ip_address,maxread=1000000,timeout=60)
@@ -190,7 +190,9 @@
'''
ssh_newkey = 'Are you sure you want to continue connecting'
refused = "ssh: connect to host "+ip_address+" port 22: Connection refused"
- self.handle =pexpect.spawn('scp '+user_name+'@'+ip_address+':'+filepath+' '+dst_path)
+ cmd = 'scp '+str(user_name)+'@'+str(ip_address)+':'+str(filepath)+' '+str(dst_path)
+ main.log.info("Sending: " + cmd )
+ self.handle =pexpect.spawn( cmd )
i=self.handle.expect([ssh_newkey,'password:',pexpect.EOF,pexpect.TIMEOUT,refused],120)
if i==0:
@@ -212,7 +214,9 @@
main.log.error("ssh: connect to host "+ip_address+" port 22: Connection refused")
return main.FALSE
- self.handle.sendline("\r")
+ self.handle.sendline("")
+ self.handle.expect("$")
+ print self.handle.before
return self.handle
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
index 0a8bd57..68f12a8 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,[6],8,3,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
@@ -28,7 +28,7 @@
<port7>6633</port7>
</CTRL>
<TESTONUSER>admin</TESTONUSER>
- <TESTONIP>10.128.30.10</TESTONIP>
+ <TESTONIP>10.128.30.9</TESTONIP>
<PING>
<source1>h8</source1>
<source2>h9</source2>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 47dc744..3e0a055 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -82,7 +82,7 @@
cell_result = main.ONOSbench.set_cell(cell_name)
verify_result = main.ONOSbench.verify_cell()
- #FIXME:this is short term fix
+ #FIXME:this is short term fix
main.log.report("Removing raft logs")
main.ONOSbench.onos_remove_raft_logs()
main.log.report("Uninstalling ONOS")
@@ -145,29 +145,32 @@
main.step("Checking if ONOS is up yet")
#TODO: Refactor
# check bundle:list?
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
- if not onos1_isup:
- main.log.report("ONOS1 didn't start!")
- onos2_isup = main.ONOSbench.isup(ONOS2_ip)
- if not onos2_isup:
- main.log.report("ONOS2 didn't start!")
- onos3_isup = main.ONOSbench.isup(ONOS3_ip)
- if not onos3_isup:
- main.log.report("ONOS3 didn't start!")
- onos4_isup = main.ONOSbench.isup(ONOS4_ip)
- if not onos4_isup:
- main.log.report("ONOS4 didn't start!")
- onos5_isup = main.ONOSbench.isup(ONOS5_ip)
- if not onos5_isup:
- main.log.report("ONOS5 didn't start!")
- onos6_isup = main.ONOSbench.isup(ONOS6_ip)
- if not onos6_isup:
- main.log.report("ONOS6 didn't start!")
- onos7_isup = main.ONOSbench.isup(ONOS7_ip)
- if not onos7_isup:
- main.log.report("ONOS7 didn't start!")
- onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
- and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ if not onos2_isup:
+ main.log.report("ONOS2 didn't start!")
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ if not onos3_isup:
+ main.log.report("ONOS3 didn't start!")
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ if not onos4_isup:
+ main.log.report("ONOS4 didn't start!")
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ if not onos5_isup:
+ main.log.report("ONOS5 didn't start!")
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ if not onos6_isup:
+ main.log.report("ONOS6 didn't start!")
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ if not onos7_isup:
+ main.log.report("ONOS7 didn't start!")
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ if onos_isup_result == main.TRUE:
+ break
# TODO: if it becomes an issue, we can retry this step a few times
@@ -197,9 +200,9 @@
onfail="Test startup NOT successful")
- #if case1_result==main.FALSE:
- # main.cleanup()
- # main.exit()
+ if case1_result==main.FALSE:
+ main.cleanup()
+ main.exit()
def CASE2(self,main) :
'''
@@ -209,25 +212,6 @@
import json
import re
-
- '''
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS1_port = main.params['CTRL']['port1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS2_port = main.params['CTRL']['port2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS3_port = main.params['CTRL']['port3']
- ONOS4_ip = main.params['CTRL']['ip4']
- ONOS4_port = main.params['CTRL']['port4']
- ONOS5_ip = main.params['CTRL']['ip5']
- ONOS5_port = main.params['CTRL']['port5']
- ONOS6_ip = main.params['CTRL']['ip6']
- ONOS6_port = main.params['CTRL']['port6']
- ONOS7_ip = main.params['CTRL']['ip7']
- ONOS7_port = main.params['CTRL']['port7']
- '''
-
-
main.log.report("Assigning switches to controllers")
main.case("Assigning Controllers")
main.step("Assign switches to controllers")
@@ -265,8 +249,217 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
+ #Manually assign mastership to the controller we want
+ role_call = main.TRUE
+ role_check = main.TRUE
+
+ device_id = main.ONOScli1.get_device("1000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2800")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("5000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3004")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+ if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3008")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3009")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3010")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3011")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3012")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3013")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3014")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3015")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3016")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3017")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6007")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+ if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6018")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6019")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6020")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6021")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6022")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6023")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6024")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6025")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6026")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6027")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_call,
+ onpass="Re-assigned switch mastership to designated controller",
+ onfail="Something wrong with device_role calls")
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_check,
+ onpass="Switches were successfully reassigned to designated controller",
+ onfail="Switches were not successfully reassigned")
+ mastership_check = mastership_check and role_call and role_check
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
def CASE3(self,main) :
@@ -274,6 +467,7 @@
Assign intents
"""
+ #FIXME: This should only be called once when we have persistence
import time
import json
import re
@@ -283,6 +477,16 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+ main.ONOScli2.feature_install("onos-app-fwd")
+ main.ONOScli3.feature_install("onos-app-fwd")
+ main.ONOScli4.feature_install("onos-app-fwd")
+ main.ONOScli5.feature_install("onos-app-fwd")
+ main.ONOScli6.feature_install("onos-app-fwd")
+ main.ONOScli7.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -304,7 +508,7 @@
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -314,7 +518,10 @@
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -360,6 +567,25 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
ONOS3_mastership = main.ONOScli3.roles()
@@ -416,6 +642,7 @@
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
ONOS2_intents = main.ONOScli2.intents( json_format=True )
ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -470,6 +697,7 @@
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
ONOS2_flows = main.ONOScli2.flows( json_format=True )
ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -527,7 +755,7 @@
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
@@ -578,7 +806,6 @@
devices.append( main.ONOScli5.devices() )
devices.append( main.ONOScli6.devices() )
devices.append( main.ONOScli7.devices() )
- '''
hosts = []
hosts.append( main.ONOScli1.hosts() )
hosts.append( main.ONOScli2.hosts() )
@@ -587,7 +814,6 @@
hosts.append( main.ONOScli5.hosts() )
hosts.append( main.ONOScli6.hosts() )
hosts.append( main.ONOScli7.hosts() )
- '''
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -604,6 +830,91 @@
links.append( main.ONOScli5.links() )
links.append( main.ONOScli6.links() )
links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
main.step("Comparing ONOS topology to MN")
@@ -639,14 +950,16 @@
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result\
+ and consistent_paths_result
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -708,6 +1021,24 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
@@ -739,7 +1070,6 @@
and ONOS1_mastership == ONOS5_mastership\
and ONOS1_mastership == ONOS6_mastership\
and ONOS1_mastership == ONOS7_mastership:
- #mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
main.log.report("Switch roles are consistent across all ONOS nodes")
else:
@@ -822,25 +1152,36 @@
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
else:
- main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS2 intents: ")
+ print json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS3 intents: ")
+ print json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS4 intents: ")
+ print json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS5 intents: ")
+ print json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS6 intents: ")
+ print json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS7 intents: ")
+ print json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost
+ '''
main.step("Compare current intents with intents before the failure")
+ #NOTE: this requires case 5 to pass for intent_state to be set.
+ # maybe we should stop the test if that fails?
if intent_state == ONOS1_intents:
same_intents = main.TRUE
main.log.report("Intents are consistent with before failure")
@@ -851,6 +1192,7 @@
onpass="Intents are consistent with before failure",
onfail="The Intents changed during failure")
intent_check = intent_check and same_intents
+ '''
@@ -859,12 +1201,12 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -880,7 +1222,7 @@
#NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range(8,18):
main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
if Loss_In_Pings == main.TRUE:
main.log.info("Loss in ping detected")
elif Loss_In_Pings == main.ERROR:
@@ -891,10 +1233,12 @@
utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
onpass="No Loss of connectivity",
onfail="Loss of dataplane connectivity detected")
+ #NOTE: Since intents are not persisted with Hazelcast, we expect this
+ Loss_In_Pings = main.FALSE
#TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -936,87 +1280,173 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
- for controller in range(7): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ elapsed = time.time() - start_time
+ cli_time = time.time() - cli_start
+ print "CLI time: " + str(cli_time)
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = topo_result and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
+
+
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+ topo_result = topo_result and int(count <= 2)
+ note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+ main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
@@ -1085,13 +1515,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -1099,7 +1534,6 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1121,9 +1555,15 @@
ip7=ONOS7_ip,port7=ONOS7_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -1137,8 +1577,26 @@
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print "Checking logs for errors on ONOS1:"
+ print main.ONOSbench.check_logs(ONOS1_ip)
+ print "Checking logs for errors on ONOS2:"
+ print main.ONOSbench.check_logs(ONOS2_ip)
+ print "Checking logs for errors on ONOS3:"
+ print main.ONOSbench.check_logs(ONOS3_ip)
+ print "Checking logs for errors on ONOS4:"
+ print main.ONOSbench.check_logs(ONOS4_ip)
+ print "Checking logs for errors on ONOS5:"
+ print main.ONOSbench.check_logs(ONOS5_ip)
+ print "Checking logs for errors on ONOS6:"
+ print main.ONOSbench.check_logs(ONOS6_ip)
+ print "Checking logs for errors on ONOS7:"
+ print main.ONOSbench.check_logs(ONOS7_ip)
+
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -1150,19 +1608,26 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
#std*.log's
@@ -1172,24 +1637,29 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
-
-
#sleep so scp can finish
time.sleep(10)
main.step("Packing and rotating pcap archives")
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
index 0a8bd57..5b97b66 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,[6],8,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
@@ -28,7 +28,7 @@
<port7>6633</port7>
</CTRL>
<TESTONUSER>admin</TESTONUSER>
- <TESTONIP>10.128.30.10</TESTONIP>
+ <TESTONIP>10.128.30.9</TESTONIP>
<PING>
<source1>h8</source1>
<source2>h9</source2>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 2af775d..3fbc29d 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -82,7 +82,7 @@
cell_result = main.ONOSbench.set_cell(cell_name)
verify_result = main.ONOSbench.verify_cell()
- #FIXME:this is short term fix
+ #FIXME:this is short term fix
main.log.report("Removing raft logs")
main.ONOSbench.onos_remove_raft_logs()
main.log.report("Uninstalling ONOS")
@@ -145,29 +145,32 @@
main.step("Checking if ONOS is up yet")
#TODO: Refactor
# check bundle:list?
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
- if not onos1_isup:
- main.log.report("ONOS1 didn't start!")
- onos2_isup = main.ONOSbench.isup(ONOS2_ip)
- if not onos2_isup:
- main.log.report("ONOS2 didn't start!")
- onos3_isup = main.ONOSbench.isup(ONOS3_ip)
- if not onos3_isup:
- main.log.report("ONOS3 didn't start!")
- onos4_isup = main.ONOSbench.isup(ONOS4_ip)
- if not onos4_isup:
- main.log.report("ONOS4 didn't start!")
- onos5_isup = main.ONOSbench.isup(ONOS5_ip)
- if not onos5_isup:
- main.log.report("ONOS5 didn't start!")
- onos6_isup = main.ONOSbench.isup(ONOS6_ip)
- if not onos6_isup:
- main.log.report("ONOS6 didn't start!")
- onos7_isup = main.ONOSbench.isup(ONOS7_ip)
- if not onos7_isup:
- main.log.report("ONOS7 didn't start!")
- onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
- and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ if not onos2_isup:
+ main.log.report("ONOS2 didn't start!")
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ if not onos3_isup:
+ main.log.report("ONOS3 didn't start!")
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ if not onos4_isup:
+ main.log.report("ONOS4 didn't start!")
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ if not onos5_isup:
+ main.log.report("ONOS5 didn't start!")
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ if not onos6_isup:
+ main.log.report("ONOS6 didn't start!")
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ if not onos7_isup:
+ main.log.report("ONOS7 didn't start!")
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ if onos_isup_result == main.TRUE:
+ break
# TODO: if it becomes an issue, we can retry this step a few times
@@ -197,9 +200,9 @@
onfail="Test startup NOT successful")
- #if case1_result==main.FALSE:
- # main.cleanup()
- # main.exit()
+ if case1_result==main.FALSE:
+ main.cleanup()
+ main.exit()
def CASE2(self,main) :
'''
@@ -209,25 +212,6 @@
import json
import re
-
- '''
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS1_port = main.params['CTRL']['port1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS2_port = main.params['CTRL']['port2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS3_port = main.params['CTRL']['port3']
- ONOS4_ip = main.params['CTRL']['ip4']
- ONOS4_port = main.params['CTRL']['port4']
- ONOS5_ip = main.params['CTRL']['ip5']
- ONOS5_port = main.params['CTRL']['port5']
- ONOS6_ip = main.params['CTRL']['ip6']
- ONOS6_port = main.params['CTRL']['port6']
- ONOS7_ip = main.params['CTRL']['ip7']
- ONOS7_port = main.params['CTRL']['port7']
- '''
-
-
main.log.report("Assigning switches to controllers")
main.case("Assigning Controllers")
main.step("Assign switches to controllers")
@@ -265,8 +249,217 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
+ #Manually assign mastership to the controller we want
+ role_call = main.TRUE
+ role_check = main.TRUE
+
+ device_id = main.ONOScli1.get_device("1000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2800")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("5000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3004")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+ if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3008")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3009")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3010")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3011")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3012")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3013")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3014")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3015")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3016")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3017")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6007")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+ if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6018")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6019")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6020")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6021")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6022")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6023")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6024")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6025")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6026")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6027")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_call,
+ onpass="Re-assigned switch mastership to designated controller",
+ onfail="Something wrong with device_role calls")
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_check,
+ onpass="Switches were successfully reassigned to designated controller",
+ onfail="Switches were not successfully reassigned")
+ mastership_check = mastership_check and role_call and role_check
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
def CASE3(self,main) :
@@ -283,6 +476,16 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+ main.ONOScli2.feature_install("onos-app-fwd")
+ main.ONOScli3.feature_install("onos-app-fwd")
+ main.ONOScli4.feature_install("onos-app-fwd")
+ main.ONOScli5.feature_install("onos-app-fwd")
+ main.ONOScli6.feature_install("onos-app-fwd")
+ main.ONOScli7.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -304,7 +507,7 @@
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -314,7 +517,10 @@
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -360,6 +566,25 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
ONOS3_mastership = main.ONOScli3.roles()
@@ -416,6 +641,7 @@
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
ONOS2_intents = main.ONOScli2.intents( json_format=True )
ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -470,6 +696,7 @@
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
ONOS2_flows = main.ONOScli2.flows( json_format=True )
ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -527,7 +754,7 @@
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
@@ -578,7 +805,6 @@
devices.append( main.ONOScli5.devices() )
devices.append( main.ONOScli6.devices() )
devices.append( main.ONOScli7.devices() )
- '''
hosts = []
hosts.append( main.ONOScli1.hosts() )
hosts.append( main.ONOScli2.hosts() )
@@ -587,7 +813,6 @@
hosts.append( main.ONOScli5.hosts() )
hosts.append( main.ONOScli6.hosts() )
hosts.append( main.ONOScli7.hosts() )
- '''
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -604,6 +829,91 @@
links.append( main.ONOScli5.links() )
links.append( main.ONOScli6.links() )
links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
main.step("Comparing ONOS topology to MN")
@@ -639,14 +949,16 @@
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result\
+ and consistent_paths_result
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -656,11 +968,14 @@
'''
The Failure case.
'''
+ import time
main.log.report("Killing 3 ONOS nodes")
main.log.case("Restart minority of ONOS nodes")
#TODO: Randomize these nodes
main.ONOSbench.onos_kill(ONOS1_ip)
+ time.sleep(10)
main.ONOSbench.onos_kill(ONOS2_ip)
+ time.sleep(10)
main.ONOSbench.onos_kill(ONOS3_ip)
main.step("Checking if ONOS is up yet")
@@ -694,6 +1009,24 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
@@ -702,20 +1035,6 @@
ONOS5_mastership = main.ONOScli5.roles()
ONOS6_mastership = main.ONOScli6.roles()
ONOS7_mastership = main.ONOScli7.roles()
- print type(ONOS1_mastership)
- print ONOS1_mastership
- print type(ONOS2_mastership)
- print ONOS2_mastership
- print type(ONOS3_mastership)
- print ONOS3_mastership
- print type(ONOS4_mastership)
- print ONOS4_mastership
- print type(ONOS5_mastership)
- print ONOS5_mastership
- print type(ONOS6_mastership)
- print ONOS6_mastership
- print type(ONOS7_mastership)
- print ONOS7_mastership
#print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
if "Error" in ONOS1_mastership or not ONOS1_mastership\
or "Error" in ONOS2_mastership or not ONOS2_mastership\
@@ -739,7 +1058,6 @@
and ONOS1_mastership == ONOS5_mastership\
and ONOS1_mastership == ONOS6_mastership\
and ONOS1_mastership == ONOS7_mastership:
- #mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
main.log.report("Switch roles are consistent across all ONOS nodes")
else:
@@ -822,25 +1140,35 @@
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
else:
- main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS2 intents: ")
+ print json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS3 intents: ")
+ print json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS4 intents: ")
+ print json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS5 intents: ")
+ print json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS6 intents: ")
+ print json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS7 intents: ")
+ print json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost
main.step("Compare current intents with intents before the failure")
+ #NOTE: this requires case 5 to pass for intent_state to be set.
+ # maybe we should stop the test if that fails?
if intent_state == ONOS1_intents:
same_intents = main.TRUE
main.log.report("Intents are consistent with before failure")
@@ -859,12 +1187,12 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -880,7 +1208,7 @@
#NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range(8,18):
main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
if Loss_In_Pings == main.TRUE:
main.log.info("Loss in ping detected")
elif Loss_In_Pings == main.ERROR:
@@ -894,7 +1222,7 @@
#TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -936,87 +1264,175 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
- for controller in range(7): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ elapsed = time.time() - start_time
+ cli_time = time.time() - cli_start
+ print "CLI time: " + str(cli_time)
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = topo_result and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ if consistent_hosts_result == main.FALSE:
+ for controller in range( len( hosts ) ):
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
+
+
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+ topo_result = topo_result and int(count <= 2)
+ note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+ main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
@@ -1085,13 +1501,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -1099,7 +1520,6 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1121,9 +1541,15 @@
ip7=ONOS7_ip,port7=ONOS7_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -1137,8 +1563,26 @@
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print "Checking logs for errors on ONOS1:"
+ print main.ONOSbench.check_logs(ONOS1_ip)
+ print "Checking logs for errors on ONOS2:"
+ print main.ONOSbench.check_logs(ONOS2_ip)
+ print "Checking logs for errors on ONOS3:"
+ print main.ONOSbench.check_logs(ONOS3_ip)
+ print "Checking logs for errors on ONOS4:"
+ print main.ONOSbench.check_logs(ONOS4_ip)
+ print "Checking logs for errors on ONOS5:"
+ print main.ONOSbench.check_logs(ONOS5_ip)
+ print "Checking logs for errors on ONOS6:"
+ print main.ONOSbench.check_logs(ONOS6_ip)
+ print "Checking logs for errors on ONOS7:"
+ print main.ONOSbench.check_logs(ONOS7_ip)
+
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -1150,19 +1594,26 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
#std*.log's
@@ -1172,24 +1623,29 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
-
-
#sleep so scp can finish
time.sleep(10)
main.step("Packing and rotating pcap archives")
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index e1e75f9..5259a47 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,[6],8,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
@@ -28,7 +28,7 @@
<port7>6633</port7>
</CTRL>
<TESTONUSER>admin</TESTONUSER>
- <TESTONIP>10.128.30.10</TESTONIP>
+ <TESTONIP>10.128.30.9</TESTONIP>
<PING>
<source1>h8</source1>
<source2>h9</source2>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 614d514..4e5b463 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -214,25 +214,6 @@
import json
import re
-
- '''
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS1_port = main.params['CTRL']['port1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS2_port = main.params['CTRL']['port2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS3_port = main.params['CTRL']['port3']
- ONOS4_ip = main.params['CTRL']['ip4']
- ONOS4_port = main.params['CTRL']['port4']
- ONOS5_ip = main.params['CTRL']['ip5']
- ONOS5_port = main.params['CTRL']['port5']
- ONOS6_ip = main.params['CTRL']['ip6']
- ONOS6_port = main.params['CTRL']['port6']
- ONOS7_ip = main.params['CTRL']['ip7']
- ONOS7_port = main.params['CTRL']['port7']
- '''
-
-
main.log.report("Assigning switches to controllers")
main.case("Assigning Controllers")
main.step("Assign switches to controllers")
@@ -270,8 +251,217 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
+ #Manually assign mastership to the controller we want
+ role_call = main.TRUE
+ role_check = main.TRUE
+
+ device_id = main.ONOScli1.get_device("1000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2800")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("5000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3004")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+ if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3008")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3009")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3010")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3011")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3012")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3013")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3014")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3015")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3016")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3017")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6007")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+ if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6018")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6019")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6020")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6021")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6022")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6023")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6024")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6025")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6026")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6027")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_call,
+ onpass="Re-assigned switch mastership to designated controller",
+ onfail="Something wrong with device_role calls")
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_check,
+ onpass="Switches were successfully reassigned to designated controller",
+ onfail="Switches were not successfully reassigned")
+ mastership_check = mastership_check and role_call and role_check
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
def CASE3(self,main) :
@@ -288,6 +478,16 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+ main.ONOScli2.feature_install("onos-app-fwd")
+ main.ONOScli3.feature_install("onos-app-fwd")
+ main.ONOScli4.feature_install("onos-app-fwd")
+ main.ONOScli5.feature_install("onos-app-fwd")
+ main.ONOScli6.feature_install("onos-app-fwd")
+ main.ONOScli7.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -309,7 +509,7 @@
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -319,7 +519,10 @@
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -365,6 +568,25 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
ONOS3_mastership = main.ONOScli3.roles()
@@ -421,6 +643,7 @@
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
ONOS2_intents = main.ONOScli2.intents( json_format=True )
ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -475,6 +698,7 @@
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
ONOS2_flows = main.ONOScli2.flows( json_format=True )
ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -532,7 +756,7 @@
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
@@ -583,7 +807,6 @@
devices.append( main.ONOScli5.devices() )
devices.append( main.ONOScli6.devices() )
devices.append( main.ONOScli7.devices() )
- '''
hosts = []
hosts.append( main.ONOScli1.hosts() )
hosts.append( main.ONOScli2.hosts() )
@@ -592,7 +815,6 @@
hosts.append( main.ONOScli5.hosts() )
hosts.append( main.ONOScli6.hosts() )
hosts.append( main.ONOScli7.hosts() )
- '''
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -609,6 +831,91 @@
links.append( main.ONOScli5.links() )
links.append( main.ONOScli6.links() )
links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
main.step("Comparing ONOS topology to MN")
@@ -644,14 +951,16 @@
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result\
+ and consistent_paths_result
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -676,6 +985,24 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
@@ -707,7 +1034,6 @@
and ONOS1_mastership == ONOS5_mastership\
and ONOS1_mastership == ONOS6_mastership\
and ONOS1_mastership == ONOS7_mastership:
- #mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
main.log.report("Switch roles are consistent across all ONOS nodes")
else:
@@ -789,25 +1115,35 @@
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
else:
- main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS2 intents: ")
+ print json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS3 intents: ")
+ print json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS4 intents: ")
+ print json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS5 intents: ")
+ print json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS6 intents: ")
+ print json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS7 intents: ")
+ print json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost
main.step("Compare current intents with intents before the failure")
+ #NOTE: this requires case 5 to pass for intent_state to be set.
+ # maybe we should stop the test if that fails?
if intent_state == ONOS1_intents:
same_intents = main.TRUE
main.log.report("Intents are consistent with before failure")
@@ -826,12 +1162,12 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -847,7 +1183,7 @@
#NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range(8,18):
main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
if Loss_In_Pings == main.TRUE:
main.log.info("Loss in ping detected")
elif Loss_In_Pings == main.ERROR:
@@ -861,7 +1197,7 @@
#TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -903,89 +1239,173 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
- print "cond 1:" + str(topo_result == main.FALSE)
- print "cond 2:" + str(elapsed < 120)
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
- for controller in range(7): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ elapsed = time.time() - start_time
+ cli_time = time.time() - cli_start
+ print "CLI time: " + str(cli_time)
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ for controller in range(7): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = topo_result and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
+
+
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+ topo_result = topo_result and int(count <= 2)
+ note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+ main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
@@ -1054,13 +1474,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -1068,7 +1493,6 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1090,9 +1514,15 @@
ip7=ONOS7_ip,port7=ONOS7_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -1106,8 +1536,26 @@
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print "Checking logs for errors on ONOS1:"
+ print main.ONOSbench.check_logs(ONOS1_ip)
+ print "Checking logs for errors on ONOS2:"
+ print main.ONOSbench.check_logs(ONOS2_ip)
+ print "Checking logs for errors on ONOS3:"
+ print main.ONOSbench.check_logs(ONOS3_ip)
+ print "Checking logs for errors on ONOS4:"
+ print main.ONOSbench.check_logs(ONOS4_ip)
+ print "Checking logs for errors on ONOS5:"
+ print main.ONOSbench.check_logs(ONOS5_ip)
+ print "Checking logs for errors on ONOS6:"
+ print main.ONOSbench.check_logs(ONOS6_ip)
+ print "Checking logs for errors on ONOS7:"
+ print main.ONOSbench.check_logs(ONOS7_ip)
+
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -1119,19 +1567,26 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
#std*.log's
@@ -1141,24 +1596,29 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
-
-
#sleep so scp can finish
time.sleep(10)
main.step("Packing and rotating pcap archives")
@@ -1169,3 +1629,22 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ #install app on onos 1
+ #wait for election
+ #check for leader
+ #install on other nodes
+ #check for leader. Should be onos1 and each app shows same leader
+ #
+
+
+
+
+
+ #Next Case
+ #add to reboot case?
+ #check for conistent leader
+ #new leader should have been elected
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
index e1e75f9..68f12a8 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
@@ -1,9 +1,9 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,[6],8,3,7,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
- <Git>True</Git>
+ <Git>False</Git>
<CTRL>
<ip1>10.128.30.11</ip1>
@@ -28,7 +28,7 @@
<port7>6633</port7>
</CTRL>
<TESTONUSER>admin</TESTONUSER>
- <TESTONIP>10.128.30.10</TESTONIP>
+ <TESTONIP>10.128.30.9</TESTONIP>
<PING>
<source1>h8</source1>
<source2>h9</source2>
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
index e6f2fde..a82bbd5 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
@@ -112,6 +112,8 @@
"clean install")
main.ONOSbench.get_version(report=True)
+ cell_result = main.ONOSbench.set_cell("SingleHA")
+ verify_result = main.ONOSbench.verify_cell()
main.step("Creating ONOS package")
package_result = main.ONOSbench.onos_package()
@@ -123,15 +125,17 @@
main.step("Checking if ONOS is up yet")
#TODO: Refactor
# check bundle:list?
- #this should be enough for ONOS to start
- time.sleep(60)
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if onos1_isup:
+ break
if not onos1_isup:
main.log.report("ONOS1 didn't start!")
+
# TODO: if it becomes an issue, we can retry this step a few times
- cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+ cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
main.step("Start Packet Capture MN")
main.Mininet2.start_tcpdump(
@@ -142,7 +146,7 @@
case1_result = (clean_install_result and package_result and
cell_result and verify_result and onos1_install_result and
- onos1_isup and cli1_results)
+ onos1_isup and cli_result)
utilities.assert_equals(expect=main.TRUE, actual=case1_result,
onpass="Test startup successful",
@@ -186,8 +190,6 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
def CASE3(self,main) :
@@ -195,6 +197,7 @@
Assign intents
"""
+ #FIXME: Only call this once when using a persistent datastore!!!!
import time
import json
import re
@@ -204,6 +207,10 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -219,7 +226,7 @@
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
@@ -229,7 +236,10 @@
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -275,6 +285,15 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ roles_not_null = main.ONOScli1.roles_not_null()
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
#print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
#TODO: Make this a meaningful check
@@ -285,14 +304,11 @@
else:
mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
- main.log.report("Switch roles are consistent across all ONOS nodes")
- utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
- onpass="Switch roles are consistent across all ONOS nodes",
- onfail="ONOS nodes have different views of switch roles")
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
intent_check = main.FALSE
if "Error" in ONOS1_intents or not ONOS1_intents:
@@ -300,14 +316,11 @@
main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
else:
intent_check = main.TRUE
- main.log.report("Intents are consistent across all ONOS nodes")
- utilities.assert_equals(expect = main.TRUE,actual=intent_check,
- onpass="Intents are consistent across all ONOS nodes",
- onfail="ONOS nodes have different views of intents")
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
flow_check = main.FALSE
if "Error" in ONOS1_flows or not ONOS1_flows:
@@ -317,42 +330,16 @@
#TODO: Do a better check, maybe compare flows on switches?
flow_state = ONOS1_flows
flow_check = main.TRUE
- main.log.report("Flow count is consistent across all ONOS nodes")
- utilities.assert_equals(expect = main.TRUE,actual=flow_check,
- onpass="The flow count is consistent across all ONOS nodes",
- onfail="ONOS nodes have different flow counts")
main.step("Get the OF Table entries")
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
- main.step("Start continuous pings")
- main.Mininet2.pingLong(src=main.params['PING']['source1'],
- target=main.params['PING']['target1'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source2'],
- target=main.params['PING']['target2'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source3'],
- target=main.params['PING']['target3'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source4'],
- target=main.params['PING']['target4'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source5'],
- target=main.params['PING']['target5'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source6'],
- target=main.params['PING']['target6'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source7'],
- target=main.params['PING']['target7'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source8'],
- target=main.params['PING']['target8'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source9'],
- target=main.params['PING']['target9'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source10'],
- target=main.params['PING']['target10'],pingTime=500)
-
main.step("Create TestONTopology object")
ctrls = []
count = 1
@@ -417,7 +404,7 @@
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -436,7 +423,7 @@
main.step("Checking if ONOS is up yet")
count = 0
- while count < 10
+ while count < 10:
onos1_isup = main.ONOSbench.isup(ONOS1_ip)
if onos1_isup == main.TRUE:
elapsed = time.time() - start
@@ -446,11 +433,12 @@
cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
- case_results = main.TRUE and onosi1_isup and cli_result
+ case_results = main.TRUE and onos1_isup and cli_result
utilities.assert_equals(expect=main.TRUE, actual=case_results,
onpass="ONOS restart successful",
onfail="ONOS restart NOT successful")
- main.log.info("ONOS took %s seconds to restart" % str(elapsed) )
+ main.log.info("ESTIMATE: ONOS took %s seconds to restart" % str(elapsed) )
+ time.sleep(5)
def CASE7(self,main) :
'''
@@ -460,6 +448,14 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ roles_not_null = main.ONOScli1.roles_not_null()
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
#FIXME: Refactor this whole case for single instance
@@ -508,13 +504,14 @@
main.log.report("Error in getting ONOS intents")
main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
else:
- intent_state = ONOS1_intents
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost
+ '''
main.step("Compare current intents with intents before the failure")
if intent_state == ONOS1_intents:
same_intents = main.TRUE
@@ -526,6 +523,7 @@
onpass="Intents are consistent with before failure",
onfail="The Intents changed during failure")
intent_check = intent_check and same_intents
+ '''
@@ -534,42 +532,20 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables")
- main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
- #FIXME: This check is always failing. Investigate cause
- #NOTE: this may be something to do with file permsissions
- # or slight change in format
- main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
- Loss_In_Pings = main.FALSE
- #NOTE: checkForLoss returns main.FALSE with 0% packet loss
- for i in range(8,18):
- main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
- if Loss_In_Pings == main.TRUE:
- main.log.info("Loss in ping detected")
- elif Loss_In_Pings == main.ERROR:
- main.log.info("There are multiple mininet process running")
- elif Loss_In_Pings == main.FALSE:
- main.log.info("No Loss in the pings")
- main.log.report("No loss of dataplane connectivity")
- utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
- onpass="No Loss of connectivity",
- onfail="Loss of dataplane connectivity detected")
-
-
#TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = mastership_check and intent_check and Flow_Tables and roles_not_null
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -606,62 +582,62 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- for controller in range(1): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ elapsed = time.time() - start_time
+ print "CLI time: " + str(time.time() - cli_start)
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ for controller in range(1): #TODO parameterize the number of controllers
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = devices_results and ports_results and links_results and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ topo_result = devices_results and ports_results and links_results
+
+ topo_result = topo_result and int(count <= 2)
+ main.log.report("Very crass estimate for topology discovery/convergence(note it takes about 1 seconds to read the topology from each ONOS instance): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
+ if elapsed > 60:
+ main.log.report("Giving up on topology convergence")
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
@@ -729,13 +705,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -743,7 +724,6 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -759,9 +739,15 @@
ip1=ONOS1_ip,port1=ONOS1_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -775,8 +761,13 @@
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print "Checking logs for errors on ONOS1:"
+ print main.ONOSbench.check_logs(ONOS1_ip)
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -788,8 +779,11 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.handle.expect("\$")
+ print main.ONOSbench.handle.before
#std*.log's
#NOTE: must end in /
@@ -798,7 +792,8 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )