Merge pull request #39 from opennetworkinglab/HA
Ha
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 3a24008..80fe0de 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -110,7 +110,7 @@
main.LogFileName = main.logdir + "/" + main.TEST + "_" +str(currentTime) + ".log"
main.ReportFileName = main.logdir + "/" + main.TEST + "_" + str(currentTime) + ".rpt"
- main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
+ main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
#### Add log-level - Report
logging.addLevelName(9, "REPORT")
@@ -229,14 +229,14 @@
#main.log.report(testResult)
main.testResult = testResult
main.log.exact(testResult)
-
+
##CSV output needed for Jenkin's plot plugin
#NOTE: the elements were orded based on the colors assigned to the data
logfile = open(main.JenkinsCSV ,"w")
logfile.write(",".join( ['Tests Failed', 'Tests Passed', 'Tests Planned'] ) + "\n")
- logfile.write(",".join( [str(main.TOTAL_TC_FAIL), str(main.TOTAL_TC_PASS), str(main.TOTAL_TC_PLANNED)] ))
- logfile.close()
-
+ logfile.write(",".join( [str(int(main.TOTAL_TC_FAIL)), str(int(main.TOTAL_TC_PASS)), str(int(main.TOTAL_TC_PLANNED))] ))
+ logfile.close()
+
diff --git a/TestON/dependencies/rotate.sh b/TestON/dependencies/rotate.sh
index 235d138..0f879b6 100755
--- a/TestON/dependencies/rotate.sh
+++ b/TestON/dependencies/rotate.sh
@@ -51,4 +51,4 @@
#Maybe this should be an argument? pack-and-rotate supports that
nr_max=20
-pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap" ${nr_max}
+pack-rotate-log ${root_dir}'/'${base_name} "${root_dir}/${base_name}*.pcap ${root_dir}/${base_name}*.log" ${nr_max}
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index 17c25a0..bf3a2c6 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -1207,7 +1207,7 @@
break
mn_ports.sort(key=float)
onos_ports.sort(key=float)
- #print "\nPorts for Switch %s:" % (switch['name'])
+ #print "\nPorts for Switch %s:" % (mn_switch['name'])
#print "\tmn_ports[] = ", mn_ports
#print "\tonos_ports[] = ", onos_ports
mn_ports_log = mn_ports
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index 1795a16..ae0f599 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -114,7 +114,7 @@
args = utilities.parse_args(["SRC","TARGET","PINGTIME"],**pingParams)
precmd = "sudo rm /tmp/ping." + args["SRC"]
self.execute(cmd=precmd,prompt="(.*)",timeout=10)
- command = "sudo mininet/util/m " + args["SRC"] + " ping "+args ["TARGET"]+" -i .2 -w " + str(args['PINGTIME']) + " > /tmp/ping." + args["SRC"] + " &"
+ command = "sudo mininet/util/m " + args["SRC"] + " ping "+args ["TARGET"]+" -i .2 -w " + str(args['PINGTIME']) + " -D > /tmp/ping." + args["SRC"] + " &"
main.log.info( command )
self.execute(cmd=command,prompt="(.*)",timeout=10)
self.handle.sendline("")
@@ -372,57 +372,40 @@
def get_flowTable(self, protoVersion, sw):
#TODO document usage
- #FIXME: clean up print statements and such
+ #TODO add option to look at cookies. ignoreing them for now
self.handle.sendline("cd")
self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
- #TODO: Write seperate versions of the function for this, possibly a string that tells it which switch is in use?
- #For 1.0 version of OVS
- #command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $6 $7 }' |sort -n -k1"
- #for 1.3 version of OVS
- #command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $3 $7 $8}' |sort -n -k1"
+ #print "get_flowTable(" + str(protoVersion) +" " + str(sw) +")"
#NOTE: Use format to force consistent flow table output across versions
if protoVersion==1.0:
- command = "sudo ovs-ofctl dump-flows " + sw + " -F OpenFlow10-table_id | awk '{OFS=\",\" ; print $1 $3 $6 $7 $8}' |sort -n -k1"
+ command = "sudo ovs-ofctl dump-flows " + sw + " -F OpenFlow10-table_id | awk '{OFS=\",\" ; print $1 $3 $6 $7 $8}' | cut -d ',' -f 2- | sort -n -k1 -r"
self.handle.sendline(command)
- self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
+ self.handle.expect(["k1 -r",pexpect.EOF,pexpect.TIMEOUT])
self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
response = self.handle.before
#print "response=", response
return response
elif protoVersion==1.3:
- command = "sudo ovs-ofctl dump-flows " + sw + " -O OpenFlow13 | awk '{OFS=\",\" ; print $1 $3 $6 $7}' |sort -n -k1"
+ command = "sudo ovs-ofctl dump-flows " + sw + " -O OpenFlow13 | awk '{OFS=\",\" ; print $1 $3 $6 $7}' | cut -d ',' -f 2- | sort -n -k1 -r"
self.handle.sendline(command)
- #print "ovs-vsctl Command sent status."
- #self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
- #print "sort return status: "
- #print self.handle.expect(["sort -n -k1",pexpect.EOF,pexpect.TIMEOUT])
- #print self.handle.before
+ self.handle.expect(["k1 -r",pexpect.EOF,pexpect.TIMEOUT])
self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
- #print "OFPST_FLOW expected status: "
- #print self.handle.expect(["OFPST_FLOW",pexpect.EOF,pexpect.TIMEOUT])
- #print self.handle.before
response = self.handle.before
#print "response=", response
- return response
-
+ return response
+ else:
+ main.log.error("Unknown protoVersion in get_flowTable(). given: ("+str(type(protoVersion))+") '"+str(protoVersion)+"'")
+
def flow_comp(self,flow1,flow2):
- #print "Inside flow compare function"
- #print "Flow1 Table:"
- #for flow in flow1:
- #print flow1
- #print "Flow2 Table"
- #for flow in flow2:
- #print flow2
-
if flow1==flow2:
return main.TRUE
else:
main.log.info("Flow tables do not match, printing tables:")
- #main.log.info("Flow Table 1:")
- #main.log.info(flow1)
- #main.log.info("Flow Table 2:")
- #main.log.info(flow2)
+ main.log.info("Flow Table 1:")
+ main.log.info(flow1)
+ main.log.info("Flow Table 2:")
+ main.log.info(flow2)
return main.FALSE
def setIpTablesOUTPUT(self, dst_ip, dst_port, action='add', packet_type='tcp',rule='DROP'):
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index a290ac3..84c4b7f 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -693,7 +693,38 @@
main.log.info(self.name+" ::::::")
main.cleanup()
main.exit()
-
+
+ def roles_not_null(self):
+ '''
+ Iterates through each device and checks if there is a master assigned
+ Returns: main.TRUE if each device has a master
+ main.FALSE any device has no master
+ '''
+ try:
+ import json
+ raw_roles = self.roles()
+ roles_json = json.loads(raw_roles)
+ #search json for the device with id then return the device
+ for device in roles_json:
+ #print device
+ if device['master'] == "none":
+ main.log.warn("Device has no master: " + str(device) )
+ return main.FALSE
+ return main.TRUE
+
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
+
+
def paths(self, src_id, dst_id):
'''
Returns string of paths, and the cost.
@@ -1716,7 +1747,7 @@
onos_node is the ip of one of the onos nodes in the cluster
role must be either master, standby, or none
- Returns main.TRUE or main.FALSE based argument varification.
+ Returns main.TRUE or main.FALSE based on argument verification.
When device-role supports errors this should be extended to
support that output
'''
@@ -1808,5 +1839,127 @@
main.cleanup()
main.exit()
+ def election_test_leader(self):
+ '''
+ * CLI command to get the current leader for the Election test application.
+ #NOTE: Requires installation of the onos-app-election feature
+ Returns: Node IP of the leader if one exists
+ None if none exists
+ Main.FALSE on error
+ '''
+ try:
+ self.handle.sendline("election-test-leader")
+ self.handle.expect("election-test-leader")
+ self.handle.expect("onos>")
+ response = self.handle.before
+ #Leader
+ node_search = re.search("The\scurrent\sleader\sfor\sthe\sElection\sapp\sis\s(?P<node>.+)\.", response)
+ if node_search:
+ node = node_search.group('node')
+ main.log.info("Election-test-leader on "+str(self.name)+" found " + node + " as the leader")
+ return node
+ #no leader
+ null_search = re.search("There\sis\scurrently\sno\sleader\selected\sfor\sthe\sElection\sapp", response)
+ if null_search:
+ main.log.info("Election-test-leader found no leader on " + self.name )
+ return None
+ #error
+ if re.search("Command\snot\sfound", response):
+ main.log.error("Election app is not loaded on " + self.name)
+ return main.FALSE
+ else:
+ main.log.error("Error in election_test_leader: unexpected response")
+ main.log.error( repr(response) )
+ return main.FALSE
+
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
+
+ def election_test_run(self):
+ '''
+ * CLI command to run for leadership of the Election test application.
+ #NOTE: Requires installation of the onos-app-election feature
+ Returns: Main.TRUE on success
+ Main.FALSE on error
+ '''
+ try:
+ self.handle.sendline("election-test-run")
+ self.handle.expect("election-test-run")
+ self.handle.expect("onos>")
+ response = self.handle.before
+ #success
+ search = re.search("Entering\sleadership\selections\sfor\sthe\sElection\sapp.", response)
+ if search:
+ main.log.info(self.name + " entering leadership elections for the Election app.")
+ return main.TRUE
+ #error
+ if re.search("Command\snot\sfound", response):
+ main.log.error("Election app is not loaded on " + self.name)
+ return main.FALSE
+ else:
+ main.log.error("Error in election_test_run: unexpected response")
+ main.log.error( repr(response) )
+ return main.FALSE
+
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
+
+ def election_test_withdraw(self):
+ '''
+ * CLI command to withdraw the local node from leadership election for
+ * the Election test application.
+ #NOTE: Requires installation of the onos-app-election feature
+ Returns: Main.TRUE on success
+ Main.FALSE on error
+ '''
+ try:
+ self.handle.sendline("election-test-withdraw")
+ self.handle.expect("election-test-withdraw")
+ self.handle.expect("onos>")
+ response = self.handle.before
+ #success
+ search = re.search("Withdrawing\sfrom\sleadership\selections\sfor\sthe\sElection\sapp.", response)
+ if search:
+ main.log.info(self.name + " withdrawing from leadership elections for the Election app.")
+ return main.TRUE
+ #error
+ if re.search("Command\snot\sfound", response):
+ main.log.error("Election app is not loaded on " + self.name)
+ return main.FALSE
+ else:
+ main.log.error("Error in election_test_withdraw: unexpected response")
+ main.log.error( repr(response) )
+ return main.FALSE
+
+
+ except pexpect.EOF:
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
+ except:
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+ main.cleanup()
+ main.exit()
#***********************************
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 418554d..678bae4 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -1072,21 +1072,21 @@
if not line.startswith("time="):
continue
#else
- print line
+ #print line
for var in line.split(","):
#print "'"+var+"'"
#print "'"+var.strip()+"'"
key, value = var.strip().split("=")
topology[key] = value
- print "topology = ", topology
+ #print "topology = ", topology
devices = topology.get('devices', False)
- print "devices = ", devices
+ #print "devices = ", devices
links = topology.get('links', False)
- print "links = ", links
- clusters = topology.get('clusters', False)
- print "clusters = ", clusters
+ #print "links = ", links
+ SCCs = topology.get('SCC(s)', False)
+ #print "SCCs = ", SCCs
paths = topology.get('paths', False)
- print "paths = ", paths
+ #print "paths = ", paths
return topology
except pexpect.EOF:
@@ -1339,3 +1339,26 @@
main.log.error( traceback.print_exc())
main.log.info(self.name+" ::::::")
+ def check_logs(self, onos_ip):
+ '''
+ runs onos-check-logs on the given onos node
+ returns the response
+ '''
+ try:
+ cmd = "onos-check-logs " + str(onos_ip)
+ self.handle.sendline(cmd)
+ self.handle.expect(cmd)
+ self.handle.expect("\$")
+ response = self.handle.before
+ return response
+ except pexpect.EOF:
+ main.log.error("Lost ssh connection")
+ main.log.error(self.name + ": EOF exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ except:
+ main.log.error("Some error in check_logs:")
+ main.log.info(self.name+" ::::::")
+ main.log.error( traceback.print_exc())
+ main.log.info(self.name+" ::::::")
+
+
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 00d00ed..878839e 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -48,7 +48,7 @@
ssh_newkey = 'Are you sure you want to continue connecting'
refused = "ssh: connect to host "+self.ip_address+" port 22: Connection refused"
if self.port:
- self.handle =pexpect.spawn('ssh -p '+self.port+' '+self.user_name+'@'+self.ip_address,maxread=50000)
+ self.handle =pexpect.spawn('ssh -p '+self.port+' '+self.user_name+'@'+self.ip_address, env = {"TERM": "vt100"} , maxread=50000)
else :
self.handle =pexpect.spawn('ssh -X '+self.user_name+'@'+self.ip_address,maxread=1000000,timeout=60)
@@ -190,7 +190,9 @@
'''
ssh_newkey = 'Are you sure you want to continue connecting'
refused = "ssh: connect to host "+ip_address+" port 22: Connection refused"
- self.handle =pexpect.spawn('scp '+user_name+'@'+ip_address+':'+filepath+' '+dst_path)
+ cmd = 'scp '+str(user_name)+'@'+str(ip_address)+':'+str(filepath)+' '+str(dst_path)
+ main.log.info("Sending: " + cmd )
+ self.handle =pexpect.spawn( cmd )
i=self.handle.expect([ssh_newkey,'password:',pexpect.EOF,pexpect.TIMEOUT,refused],120)
if i==0:
@@ -212,7 +214,9 @@
main.log.error("ssh: connect to host "+ip_address+" port 22: Connection refused")
return main.FALSE
- self.handle.sendline("\r")
+ self.handle.sendline("")
+ self.handle.expect("$")
+ print self.handle.before
return self.handle
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
index 0a8bd57..38a9b02 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -1,9 +1,10 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
<Git>False</Git>
+ <num_controllers> 7 </num_controllers>
<CTRL>
<ip1>10.128.30.11</ip1>
@@ -28,7 +29,7 @@
<port7>6633</port7>
</CTRL>
<TESTONUSER>admin</TESTONUSER>
- <TESTONIP>10.128.30.10</TESTONIP>
+ <TESTONIP>10.128.30.9</TESTONIP>
<PING>
<source1>h8</source1>
<source2>h9</source2>
@@ -52,8 +53,8 @@
<target10>10.0.0.27</target10>
</PING>
<timers>
- <LinkDiscovery>2</LinkDiscovery>
- <SwitchDiscovery>2</SwitchDiscovery>
+ <LinkDiscovery>.2</LinkDiscovery>
+ <SwitchDiscovery>.2</SwitchDiscovery>
</timers>
<MNtcpdump>
<intf>eth0</intf>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 47dc744..b9a07b9 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -16,6 +16,8 @@
CASE11: Switch down
CASE12: Switch up
CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
'''
class HATestClusterRestart:
@@ -61,6 +63,7 @@
global ONOS6_port
global ONOS7_ip
global ONOS7_port
+ global num_controllers
ONOS1_ip = main.params['CTRL']['ip1']
ONOS1_port = main.params['CTRL']['port1']
@@ -76,13 +79,14 @@
ONOS6_port = main.params['CTRL']['port6']
ONOS7_ip = main.params['CTRL']['ip7']
ONOS7_port = main.params['CTRL']['port7']
+ num_controllers = int(main.params['num_controllers'])
main.step("Applying cell variable to environment")
cell_result = main.ONOSbench.set_cell(cell_name)
verify_result = main.ONOSbench.verify_cell()
- #FIXME:this is short term fix
+ #FIXME:this is short term fix
main.log.report("Removing raft logs")
main.ONOSbench.onos_remove_raft_logs()
main.log.report("Uninstalling ONOS")
@@ -143,31 +147,33 @@
main.step("Checking if ONOS is up yet")
- #TODO: Refactor
- # check bundle:list?
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
- if not onos1_isup:
- main.log.report("ONOS1 didn't start!")
- onos2_isup = main.ONOSbench.isup(ONOS2_ip)
- if not onos2_isup:
- main.log.report("ONOS2 didn't start!")
- onos3_isup = main.ONOSbench.isup(ONOS3_ip)
- if not onos3_isup:
- main.log.report("ONOS3 didn't start!")
- onos4_isup = main.ONOSbench.isup(ONOS4_ip)
- if not onos4_isup:
- main.log.report("ONOS4 didn't start!")
- onos5_isup = main.ONOSbench.isup(ONOS5_ip)
- if not onos5_isup:
- main.log.report("ONOS5 didn't start!")
- onos6_isup = main.ONOSbench.isup(ONOS6_ip)
- if not onos6_isup:
- main.log.report("ONOS6 didn't start!")
- onos7_isup = main.ONOSbench.isup(ONOS7_ip)
- if not onos7_isup:
- main.log.report("ONOS7 didn't start!")
- onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
- and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ #TODO check bundle:list?
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ if not onos2_isup:
+ main.log.report("ONOS2 didn't start!")
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ if not onos3_isup:
+ main.log.report("ONOS3 didn't start!")
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ if not onos4_isup:
+ main.log.report("ONOS4 didn't start!")
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ if not onos5_isup:
+ main.log.report("ONOS5 didn't start!")
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ if not onos6_isup:
+ main.log.report("ONOS6 didn't start!")
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ if not onos7_isup:
+ main.log.report("ONOS7 didn't start!")
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ if onos_isup_result == main.TRUE:
+ break
# TODO: if it becomes an issue, we can retry this step a few times
@@ -197,9 +203,9 @@
onfail="Test startup NOT successful")
- #if case1_result==main.FALSE:
- # main.cleanup()
- # main.exit()
+ if case1_result==main.FALSE:
+ main.cleanup()
+ main.exit()
def CASE2(self,main) :
'''
@@ -209,31 +215,12 @@
import json
import re
-
- '''
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS1_port = main.params['CTRL']['port1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS2_port = main.params['CTRL']['port2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS3_port = main.params['CTRL']['port3']
- ONOS4_ip = main.params['CTRL']['ip4']
- ONOS4_port = main.params['CTRL']['port4']
- ONOS5_ip = main.params['CTRL']['ip5']
- ONOS5_port = main.params['CTRL']['port5']
- ONOS6_ip = main.params['CTRL']['ip6']
- ONOS6_port = main.params['CTRL']['port6']
- ONOS7_ip = main.params['CTRL']['ip7']
- ONOS7_port = main.params['CTRL']['port7']
- '''
-
-
main.log.report("Assigning switches to controllers")
main.case("Assigning Controllers")
main.step("Assign switches to controllers")
for i in range (1,29):
- main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+ main.Mininet1.assign_sw_controller(sw=str(i),count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -265,8 +252,217 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
+ #Manually assign mastership to the controller we want
+ role_call = main.TRUE
+ role_check = main.TRUE
+
+ device_id = main.ONOScli1.get_device("1000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2800")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("5000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3004")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+ if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3008")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3009")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3010")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3011")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3012")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3013")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3014")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3015")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3016")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3017")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6007")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+ if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6018")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6019")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6020")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6021")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6022")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6023")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6024")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6025")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6026")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6027")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_call,
+ onpass="Re-assigned switch mastership to designated controller",
+ onfail="Something wrong with device_role calls")
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_check,
+ onpass="Switches were successfully reassigned to designated controller",
+ onfail="Switches were not successfully reassigned")
+ mastership_check = mastership_check and role_call and role_check
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
def CASE3(self,main) :
@@ -274,6 +470,7 @@
Assign intents
"""
+ #FIXME: we must reinstall intents until we have a persistant datastore!
import time
import json
import re
@@ -283,6 +480,16 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+ main.ONOScli2.feature_install("onos-app-fwd")
+ main.ONOScli3.feature_install("onos-app-fwd")
+ main.ONOScli4.feature_install("onos-app-fwd")
+ main.ONOScli5.feature_install("onos-app-fwd")
+ main.ONOScli6.feature_install("onos-app-fwd")
+ main.ONOScli7.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -299,22 +506,30 @@
main.ONOScli5.feature_uninstall("onos-app-fwd")
main.ONOScli6.feature_uninstall("onos-app-fwd")
main.ONOScli7.feature_uninstall("onos-app-fwd")
+ #timeout for fwd flows
+ time.sleep(10)
main.step("Add host intents")
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
- #NOTE: get host can return None
- #TODO: handle this
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
- tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ #NOTE: get host can return None
+ if host1_id and host2_id:
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ else:
+ main.log.error("Error, get_host() failed")
+ tmp_result = main.FALSE
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -360,6 +575,25 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
ONOS3_mastership = main.ONOScli3.roles()
@@ -416,6 +650,7 @@
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
ONOS2_intents = main.ONOScli2.intents( json_format=True )
ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -470,6 +705,7 @@
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
ONOS2_flows = main.ONOScli2.flows( json_format=True )
ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -527,7 +763,7 @@
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
@@ -578,7 +814,6 @@
devices.append( main.ONOScli5.devices() )
devices.append( main.ONOScli6.devices() )
devices.append( main.ONOScli7.devices() )
- '''
hosts = []
hosts.append( main.ONOScli1.hosts() )
hosts.append( main.ONOScli2.hosts() )
@@ -587,7 +822,6 @@
hosts.append( main.ONOScli5.hosts() )
hosts.append( main.ONOScli6.hosts() )
hosts.append( main.ONOScli7.hosts() )
- '''
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -604,13 +838,98 @@
links.append( main.ONOScli5.links() )
links.append( main.ONOScli6.links() )
links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
main.step("Comparing ONOS topology to MN")
devices_results = main.TRUE
ports_results = main.TRUE
links_results = main.TRUE
- for controller in range(7): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -639,14 +958,16 @@
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result\
+ and consistent_paths_result
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -708,6 +1029,24 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
@@ -739,7 +1078,6 @@
and ONOS1_mastership == ONOS5_mastership\
and ONOS1_mastership == ONOS6_mastership\
and ONOS1_mastership == ONOS7_mastership:
- #mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
main.log.report("Switch roles are consistent across all ONOS nodes")
else:
@@ -785,7 +1123,7 @@
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed")
#NOTE: we expect mastership to change on controller failure
- mastership_check = mastership_check #and consistent_mastership
+ mastership_check = consistent_mastership
@@ -822,35 +1160,53 @@
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
else:
- main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS2 intents: ")
+ print json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS3 intents: ")
+ print json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS4 intents: ")
+ print json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS5 intents: ")
+ print json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS6 intents: ")
+ print json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS7 intents: ")
+ print json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost across system restarts
+ '''
main.step("Compare current intents with intents before the failure")
+ #NOTE: this requires case 5 to pass for intent_state to be set.
+ # maybe we should stop the test if that fails?
if intent_state == ONOS1_intents:
same_intents = main.TRUE
main.log.report("Intents are consistent with before failure")
#TODO: possibly the states have changed? we may need to figure out what the aceptable states are
else:
+ try:
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ except:
+ pass
same_intents = main.FALSE
utilities.assert_equals(expect = main.TRUE,actual=same_intents,
onpass="Intents are consistent with before failure",
onfail="The Intents changed during failure")
intent_check = intent_check and same_intents
+ '''
@@ -859,12 +1215,12 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -880,7 +1236,7 @@
#NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range(8,18):
main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
if Loss_In_Pings == main.TRUE:
main.log.info("Loss in ping detected")
elif Loss_In_Pings == main.ERROR:
@@ -891,10 +1247,37 @@
utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
onpass="No Loss of connectivity",
onfail="Loss of dataplane connectivity detected")
+ #NOTE: Since intents are not persisted with Hazelcast, we expect this
+ Loss_In_Pings = main.FALSE
- #TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ #Test of LeadershipElection
+ leader_list = []
+ leader_result = main.TRUE
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leaderN = node.election_test_leader()
+ leader_list.append(leaderN)
+ if leaderN == main.FALSE:
+ #error in response
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leaderN == None:
+ main.log.report("ONOS"+str(controller) + " shows no leader for the election-app was elected after the old one died")
+ leader_result = main.FALSE
+ if len( set( leader_list ) ) != 1:
+ leader_result = main.FALSE
+ main.log.error("Inconsistent view of leader for the election test app")
+ #TODO: print the list
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected after restart)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null\
+ and leader_result
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -936,87 +1319,173 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
- for controller in range(7): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ elapsed = time.time() - start_time
+ cli_time = time.time() - cli_start
+ print "CLI time: " + str(cli_time)
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ for controller in range(num_controllers):
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = topo_result and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
+
+
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+ topo_result = topo_result and int(count <= 2)
+ note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+ main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
@@ -1031,7 +1500,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Turn off a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1053,7 +1522,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Restore a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1076,7 +1545,7 @@
#NOTE: You should probably run a topology check after this
import time
- switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Killing a switch to ensure it is discovered correctly"
main.log.report(description)
@@ -1085,13 +1554,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -1099,7 +1573,8 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1111,7 +1586,7 @@
main.Mininet1.add_link('s28', 's3')
main.Mininet1.add_link('s28', 's6')
main.Mininet1.add_link('s28', 'h28')
- main.Mininet1.assign_sw_controller(sw="28",count=7,
+ main.Mininet1.assign_sw_controller(sw="28",count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -1121,9 +1596,15 @@
ip7=ONOS7_ip,port7=ONOS7_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -1131,14 +1612,41 @@
'''
import os
import time
+ #printing colors to terminal
+ colors = {}
+ colors['cyan'] = '\033[96m'
+ colors['purple'] = '\033[95m'
+ colors['blue'] = '\033[94m'
+ colors['green'] = '\033[92m'
+ colors['yellow'] = '\033[93m'
+ colors['red'] = '\033[91m'
+ colors['end'] = '\033[0m'
description = "Test Cleanup"
main.log.report(description)
main.case(description)
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS1_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS2:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS2_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS3:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS3_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS4:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS4_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS5:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS5_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS6:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS6_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS7:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS7_ip)
+
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -1150,19 +1658,26 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
#std*.log's
@@ -1172,24 +1687,29 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
-
-
#sleep so scp can finish
time.sleep(10)
main.step("Packing and rotating pcap archives")
@@ -1200,3 +1720,139 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ leader_result = main.TRUE
+ #install app on onos 1
+ main.log.info("Install leadership election app")
+ main.ONOScli1.feature_install("onos-app-election")
+ #wait for election
+ #check for leader
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ #all is well
+ pass
+ elif leader == None:
+ #No leader elected
+ main.log.report("No leader was elected")
+ leader_result = main.FALSE
+ elif leader == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ else:
+ #error in response
+ main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+ leader_result = main.FALSE
+
+
+
+
+ #install on other nodes and check for leader.
+ #Should be onos1 and each app should show the same leader
+ for controller in range(2,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ node.feature_install("onos-app-election")
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == ONOS1_ip:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+ def CASE15 ( self, main ) :
+ '''
+ Check that Leadership Election is still functional
+ '''
+ leader_result = main.TRUE
+ description = "Check that Leadership Election is still functional"
+ main.log.report(description)
+ main.case(description)
+ main.step("Find current leader and withdraw")
+ leader = main.ONOScli1.election_test_leader()
+ #TODO: do some sanity checking on leader before using it
+ withdraw_result = main.FALSE
+ if leader == ONOS1_ip:
+ old_leader = getattr( main, "ONOScli1" )
+ elif leader == ONOS2_ip:
+ old_leader = getattr( main, "ONOScli2" )
+ elif leader == ONOS3_ip:
+ old_leader = getattr( main, "ONOScli3" )
+ elif leader == ONOS4_ip:
+ old_leader = getattr( main, "ONOScli4" )
+ elif leader == ONOS5_ip:
+ old_leader = getattr( main, "ONOScli5" )
+ elif leader == ONOS6_ip:
+ old_leader = getattr( main, "ONOScli6" )
+ elif leader == ONOS7_ip:
+ old_leader = getattr( main, "ONOScli7" )
+ elif leader == None or leader == main.FALSE:
+ main.log.report("Leader for the election app should be an ONOS node,"\
+ +"instead got '"+str(leader)+"'")
+ leader_result = main.FALSE
+ withdraw_result = old_leader.election_test_withdraw()
+
+
+ main.step("Make sure new leader is elected")
+ leader_list = []
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leader_list.append( node.election_test_leader() )
+ for leaderN in leader_list:
+ if leaderN == leader:
+ main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+ " as leader after they withdrew")
+ leader_result = main.FALSE
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ consistent_leader = main.FALSE
+ if len( set( leader_list ) ) == 1:
+ main.log.info("Each Election-app sees '"+str(leader_list[0])+"' as the leader")
+ consistent_leader = main.TRUE
+ else:
+ main.log.report("Inconsistent responses for leader of Election-app:")
+ for n in range(len(leader_list)):
+ main.log.report("ONOS" + str(n+1) + " response: " + str(leader_list[n]) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was elected when the old leader resigned)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ main.step("Run for election on old leader(just so everyone is in the hat)")
+ run_result = old_leader.election_test_run()
+ if consistent_leader == main.TRUE:
+ after_run = main.ONOScli1.election_test_leader()
+ #verify leader didn't just change
+ if after_run == leader_list[0]:
+ leader_result = main.TRUE
+ else:
+ leader_result = main.FALSE
+ #TODO: assert on run and withdraw results?
+
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election after the old leader re-ran for election")
+
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
index 0a8bd57..78e0c83 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -1,9 +1,10 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
<Git>False</Git>
+ <num_controllers> 7 </num_controllers>
<CTRL>
<ip1>10.128.30.11</ip1>
@@ -28,7 +29,7 @@
<port7>6633</port7>
</CTRL>
<TESTONUSER>admin</TESTONUSER>
- <TESTONIP>10.128.30.10</TESTONIP>
+ <TESTONIP>10.128.30.9</TESTONIP>
<PING>
<source1>h8</source1>
<source2>h9</source2>
@@ -52,8 +53,8 @@
<target10>10.0.0.27</target10>
</PING>
<timers>
- <LinkDiscovery>2</LinkDiscovery>
- <SwitchDiscovery>2</SwitchDiscovery>
+ <LinkDiscovery>.2</LinkDiscovery>
+ <SwitchDiscovery>.2</SwitchDiscovery>
</timers>
<MNtcpdump>
<intf>eth0</intf>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 2af775d..b9d2925 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -16,6 +16,8 @@
CASE11: Switch down
CASE12: Switch up
CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
'''
class HATestMinorityRestart:
@@ -61,6 +63,7 @@
global ONOS6_port
global ONOS7_ip
global ONOS7_port
+ global num_controllers
ONOS1_ip = main.params['CTRL']['ip1']
ONOS1_port = main.params['CTRL']['port1']
@@ -76,13 +79,14 @@
ONOS6_port = main.params['CTRL']['port6']
ONOS7_ip = main.params['CTRL']['ip7']
ONOS7_port = main.params['CTRL']['port7']
+ num_controllers = int(main.params['num_controllers'])
main.step("Applying cell variable to environment")
cell_result = main.ONOSbench.set_cell(cell_name)
verify_result = main.ONOSbench.verify_cell()
- #FIXME:this is short term fix
+ #FIXME:this is short term fix
main.log.report("Removing raft logs")
main.ONOSbench.onos_remove_raft_logs()
main.log.report("Uninstalling ONOS")
@@ -143,31 +147,33 @@
main.step("Checking if ONOS is up yet")
- #TODO: Refactor
- # check bundle:list?
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
- if not onos1_isup:
- main.log.report("ONOS1 didn't start!")
- onos2_isup = main.ONOSbench.isup(ONOS2_ip)
- if not onos2_isup:
- main.log.report("ONOS2 didn't start!")
- onos3_isup = main.ONOSbench.isup(ONOS3_ip)
- if not onos3_isup:
- main.log.report("ONOS3 didn't start!")
- onos4_isup = main.ONOSbench.isup(ONOS4_ip)
- if not onos4_isup:
- main.log.report("ONOS4 didn't start!")
- onos5_isup = main.ONOSbench.isup(ONOS5_ip)
- if not onos5_isup:
- main.log.report("ONOS5 didn't start!")
- onos6_isup = main.ONOSbench.isup(ONOS6_ip)
- if not onos6_isup:
- main.log.report("ONOS6 didn't start!")
- onos7_isup = main.ONOSbench.isup(ONOS7_ip)
- if not onos7_isup:
- main.log.report("ONOS7 didn't start!")
- onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
- and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ #TODO check bundle:list?
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ if not onos2_isup:
+ main.log.report("ONOS2 didn't start!")
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ if not onos3_isup:
+ main.log.report("ONOS3 didn't start!")
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ if not onos4_isup:
+ main.log.report("ONOS4 didn't start!")
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ if not onos5_isup:
+ main.log.report("ONOS5 didn't start!")
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ if not onos6_isup:
+ main.log.report("ONOS6 didn't start!")
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ if not onos7_isup:
+ main.log.report("ONOS7 didn't start!")
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ if onos_isup_result == main.TRUE:
+ break
# TODO: if it becomes an issue, we can retry this step a few times
@@ -181,6 +187,7 @@
cli_results = cli_result1 and cli_result2 and cli_result3 and\
cli_result4 and cli_result5 and cli_result6 and cli_result7
+
main.step("Start Packet Capture MN")
main.Mininet2.start_tcpdump(
str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
@@ -197,9 +204,9 @@
onfail="Test startup NOT successful")
- #if case1_result==main.FALSE:
- # main.cleanup()
- # main.exit()
+ if case1_result==main.FALSE:
+ main.cleanup()
+ main.exit()
def CASE2(self,main) :
'''
@@ -209,31 +216,12 @@
import json
import re
-
- '''
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS1_port = main.params['CTRL']['port1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS2_port = main.params['CTRL']['port2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS3_port = main.params['CTRL']['port3']
- ONOS4_ip = main.params['CTRL']['ip4']
- ONOS4_port = main.params['CTRL']['port4']
- ONOS5_ip = main.params['CTRL']['ip5']
- ONOS5_port = main.params['CTRL']['port5']
- ONOS6_ip = main.params['CTRL']['ip6']
- ONOS6_port = main.params['CTRL']['port6']
- ONOS7_ip = main.params['CTRL']['ip7']
- ONOS7_port = main.params['CTRL']['port7']
- '''
-
-
main.log.report("Assigning switches to controllers")
main.case("Assigning Controllers")
main.step("Assign switches to controllers")
for i in range (1,29):
- main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+ main.Mininet1.assign_sw_controller(sw=str(i),count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -265,8 +253,217 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
+ #Manually assign mastership to the controller we want
+ role_call = main.TRUE
+ role_check = main.TRUE
+
+ device_id = main.ONOScli1.get_device("1000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2800")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("5000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3004")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+ if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3008")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3009")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3010")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3011")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3012")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3013")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3014")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3015")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3016")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3017")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6007")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+ if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6018")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6019")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6020")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6021")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6022")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6023")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6024")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6025")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6026")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6027")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_call,
+ onpass="Re-assigned switch mastership to designated controller",
+ onfail="Something wrong with device_role calls")
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_check,
+ onpass="Switches were successfully reassigned to designated controller",
+ onfail="Switches were not successfully reassigned")
+ mastership_check = mastership_check and role_call and role_check
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
def CASE3(self,main) :
@@ -283,6 +480,16 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+ main.ONOScli2.feature_install("onos-app-fwd")
+ main.ONOScli3.feature_install("onos-app-fwd")
+ main.ONOScli4.feature_install("onos-app-fwd")
+ main.ONOScli5.feature_install("onos-app-fwd")
+ main.ONOScli6.feature_install("onos-app-fwd")
+ main.ONOScli7.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -299,22 +506,30 @@
main.ONOScli5.feature_uninstall("onos-app-fwd")
main.ONOScli6.feature_uninstall("onos-app-fwd")
main.ONOScli7.feature_uninstall("onos-app-fwd")
+ #timeout for fwd flows
+ time.sleep(10)
main.step("Add host intents")
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
- #NOTE: get host can return None
- #TODO: handle this
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
- tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ #NOTE: get host can return None
+ if host1_id and host2_id:
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ else:
+ main.log.error("Error, get_host() failed")
+ tmp_result = main.FALSE
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -360,6 +575,25 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
ONOS3_mastership = main.ONOScli3.roles()
@@ -416,6 +650,7 @@
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
ONOS2_intents = main.ONOScli2.intents( json_format=True )
ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -470,6 +705,7 @@
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
ONOS2_flows = main.ONOScli2.flows( json_format=True )
ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -527,7 +763,7 @@
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
@@ -578,16 +814,18 @@
devices.append( main.ONOScli5.devices() )
devices.append( main.ONOScli6.devices() )
devices.append( main.ONOScli7.devices() )
- '''
hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
+ hosts.append( json.loads( main.ONOScli1.hosts() ) )
+ hosts.append( json.loads( main.ONOScli2.hosts() ) )
+ hosts.append( json.loads( main.ONOScli3.hosts() ) )
+ hosts.append( json.loads( main.ONOScli4.hosts() ) )
+ hosts.append( json.loads( main.ONOScli5.hosts() ) )
+ hosts.append( json.loads( main.ONOScli6.hosts() ) )
+ hosts.append( json.loads( main.ONOScli7.hosts() ) )
+ for controller in range(0, len(hosts) ):
+ for host in hosts[controller]:
+ if host['ips'] == []:
+ main.log.error("DEBUG:Error with host ips on controller"+str(controller+1)+": " + str(host))
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -604,13 +842,98 @@
links.append( main.ONOScli5.links() )
links.append( main.ONOScli6.links() )
links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
main.step("Comparing ONOS topology to MN")
devices_results = main.TRUE
ports_results = main.TRUE
links_results = main.TRUE
- for controller in range(7): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -639,14 +962,16 @@
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result\
+ and consistent_paths_result
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -656,11 +981,14 @@
'''
The Failure case.
'''
+ import time
main.log.report("Killing 3 ONOS nodes")
main.log.case("Restart minority of ONOS nodes")
#TODO: Randomize these nodes
main.ONOSbench.onos_kill(ONOS1_ip)
+ time.sleep(10)
main.ONOSbench.onos_kill(ONOS2_ip)
+ time.sleep(10)
main.ONOSbench.onos_kill(ONOS3_ip)
main.step("Checking if ONOS is up yet")
@@ -680,6 +1008,8 @@
cli_result3 = main.ONOScli3.start_onos_cli(ONOS3_ip)
cli_results = cli_result1 and cli_result2 and cli_result3
+ main.log.info("Install leadership election app on restarted node")
+
case_results = main.TRUE and onos_isup_result and cli_results
utilities.assert_equals(expect=main.TRUE, actual=case_results,
onpass="ONOS restart successful",
@@ -694,6 +1024,24 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
@@ -702,20 +1050,6 @@
ONOS5_mastership = main.ONOScli5.roles()
ONOS6_mastership = main.ONOScli6.roles()
ONOS7_mastership = main.ONOScli7.roles()
- print type(ONOS1_mastership)
- print ONOS1_mastership
- print type(ONOS2_mastership)
- print ONOS2_mastership
- print type(ONOS3_mastership)
- print ONOS3_mastership
- print type(ONOS4_mastership)
- print ONOS4_mastership
- print type(ONOS5_mastership)
- print ONOS5_mastership
- print type(ONOS6_mastership)
- print ONOS6_mastership
- print type(ONOS7_mastership)
- print ONOS7_mastership
#print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
if "Error" in ONOS1_mastership or not ONOS1_mastership\
or "Error" in ONOS2_mastership or not ONOS2_mastership\
@@ -739,7 +1073,6 @@
and ONOS1_mastership == ONOS5_mastership\
and ONOS1_mastership == ONOS6_mastership\
and ONOS1_mastership == ONOS7_mastership:
- #mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
main.log.report("Switch roles are consistent across all ONOS nodes")
else:
@@ -785,7 +1118,7 @@
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed")
#NOTE: we expect mastership to change on controller failure
- mastership_check = mastership_check #and consistent_mastership
+ mastership_check = consistent_mastership
@@ -822,30 +1155,46 @@
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
else:
- main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS2 intents: ")
+ print json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS3 intents: ")
+ print json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS4 intents: ")
+ print json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS5 intents: ")
+ print json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS6 intents: ")
+ print json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS7 intents: ")
+ print json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost across system restarts
main.step("Compare current intents with intents before the failure")
+ #NOTE: this requires case 5 to pass for intent_state to be set.
+ # maybe we should stop the test if that fails?
if intent_state == ONOS1_intents:
same_intents = main.TRUE
main.log.report("Intents are consistent with before failure")
#TODO: possibly the states have changed? we may need to figure out what the aceptable states are
else:
+ try:
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ except:
+ pass
same_intents = main.FALSE
utilities.assert_equals(expect = main.TRUE,actual=same_intents,
onpass="Intents are consistent with before failure",
@@ -859,12 +1208,12 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -880,7 +1229,7 @@
#NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range(8,18):
main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
if Loss_In_Pings == main.TRUE:
main.log.info("Loss in ping detected")
elif Loss_In_Pings == main.ERROR:
@@ -893,8 +1242,36 @@
onfail="Loss of dataplane connectivity detected")
- #TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ #Test of LeadershipElection
+ leader_list = []
+ leader_result = main.TRUE
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leaderN = node.election_test_leader()
+ leader_list.append(leaderN)
+ if leaderN == main.FALSE:
+ #error in response
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leaderN == None:
+ main.log.report("ONOS"+str(controller) + " shows no leader for the election-app was elected after the old one died")
+ leader_result = main.FALSE
+ elif leaderN == ONOS1_ip or leaderN == ONOS2_ip or leaderN == ONOS3_ip:
+ main.log.report("ONOS"+str(controller) + " shows "+str(leaderN)+" as leader for the election-app, but it was restarted")
+ leader_result = main.FALSE
+ if len( set( leader_list ) ) != 1:
+ leader_result = main.FALSE
+ main.log.error("Inconsistent view of leader for the election test app")
+ #TODO: print the list
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null\
+ and leader_result
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -936,87 +1313,180 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ hosts = []
+ hosts.append( json.loads( main.ONOScli1.hosts() ) )
+ hosts.append( json.loads( main.ONOScli2.hosts() ) )
+ hosts.append( json.loads( main.ONOScli3.hosts() ) )
+ hosts.append( json.loads( main.ONOScli4.hosts() ) )
+ hosts.append( json.loads( main.ONOScli5.hosts() ) )
+ hosts.append( json.loads( main.ONOScli6.hosts() ) )
+ hosts.append( json.loads( main.ONOScli7.hosts() ) )
+ for controller in range(0, len(hosts) ):
+ for host in hosts[controller]:
+ host
+ if host['ips'] == []:
+ main.log.error("DEBUG:Error with host ips on controller"+str(controller+1)+": " + str(host))
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
- for controller in range(7): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ elapsed = time.time() - start_time
+ cli_time = time.time() - cli_start
+ print "CLI time: " + str(cli_time)
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ for controller in range(num_controllers):
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = topo_result and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ if consistent_hosts_result == main.FALSE:
+ for controller in range( len( hosts ) ):
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
+
+
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+ topo_result = topo_result and int(count <= 2)
+ note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+ main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
@@ -1031,7 +1501,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Turn off a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1053,7 +1523,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Restore a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1076,7 +1546,7 @@
#NOTE: You should probably run a topology check after this
import time
- switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Killing a switch to ensure it is discovered correctly"
main.log.report(description)
@@ -1085,13 +1555,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -1099,7 +1574,8 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1111,7 +1587,7 @@
main.Mininet1.add_link('s28', 's3')
main.Mininet1.add_link('s28', 's6')
main.Mininet1.add_link('s28', 'h28')
- main.Mininet1.assign_sw_controller(sw="28",count=7,
+ main.Mininet1.assign_sw_controller(sw="28",count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -1121,9 +1597,15 @@
ip7=ONOS7_ip,port7=ONOS7_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -1131,14 +1613,41 @@
'''
import os
import time
+ #printing colors to terminal
+ colors = {}
+ colors['cyan'] = '\033[96m'
+ colors['purple'] = '\033[95m'
+ colors['blue'] = '\033[94m'
+ colors['green'] = '\033[92m'
+ colors['yellow'] = '\033[93m'
+ colors['red'] = '\033[91m'
+ colors['end'] = '\033[0m'
description = "Test Cleanup"
main.log.report(description)
main.case(description)
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS1_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS2:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS2_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS3:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS3_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS4:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS4_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS5:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS5_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS6:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS6_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS7:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS7_ip)
+
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -1150,19 +1659,26 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
#std*.log's
@@ -1172,24 +1688,29 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
-
-
#sleep so scp can finish
time.sleep(10)
main.step("Packing and rotating pcap archives")
@@ -1200,3 +1721,139 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ leader_result = main.TRUE
+ #install app on onos 1
+ main.log.info("Install leadership election app")
+ main.ONOScli1.feature_install("onos-app-election")
+ #wait for election
+ #check for leader
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ #all is well
+ pass
+ elif leader == None:
+ #No leader elected
+ main.log.report("No leader was elected")
+ leader_result = main.FALSE
+ elif leader == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ else:
+ #error in response
+ main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+ leader_result = main.FALSE
+
+
+
+
+ #install on other nodes and check for leader.
+ #Should be onos1 and each app should show the same leader
+ for controller in range(2,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ node.feature_install("onos-app-election")
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == ONOS1_ip:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+ def CASE15 ( self, main ) :
+ '''
+ Check that Leadership Election is still functional
+ '''
+ leader_result = main.TRUE
+ description = "Check that Leadership Election is still functional"
+ main.log.report(description)
+ main.case(description)
+ main.step("Find current leader and withdraw")
+ leader = main.ONOScli1.election_test_leader()
+ #TODO: do some sanity checking on leader before using it
+ withdraw_result = main.FALSE
+ if leader == ONOS1_ip:
+ old_leader = getattr( main, "ONOScli1" )
+ elif leader == ONOS2_ip:
+ old_leader = getattr( main, "ONOScli2" )
+ elif leader == ONOS3_ip:
+ old_leader = getattr( main, "ONOScli3" )
+ elif leader == ONOS4_ip:
+ old_leader = getattr( main, "ONOScli4" )
+ elif leader == ONOS5_ip:
+ old_leader = getattr( main, "ONOScli5" )
+ elif leader == ONOS6_ip:
+ old_leader = getattr( main, "ONOScli6" )
+ elif leader == ONOS7_ip:
+ old_leader = getattr( main, "ONOScli7" )
+ elif leader == None or leader == main.FALSE:
+ main.log.report("Leader for the election app should be an ONOS node,"\
+ +"instead got '"+str(leader)+"'")
+ leader_result = main.FALSE
+ withdraw_result = old_leader.election_test_withdraw()
+
+
+ main.step("Make sure new leader is elected")
+ leader_list = []
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leader_list.append( node.election_test_leader() )
+ for leaderN in leader_list:
+ if leaderN == leader:
+ main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+ " as leader after they withdrew")
+ leader_result = main.FALSE
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ consistent_leader = main.FALSE
+ if len( set( leader_list ) ) == 1:
+ main.log.info("Each Election-app sees '"+str(leader_list[0])+"' as the leader")
+ consistent_leader = main.TRUE
+ else:
+ main.log.report("Inconsistent responses for leader of Election-app:")
+ for n in range(len(leader_list)):
+ main.log.report("ONOS" + str(n+1) + " response: " + str(leader_list[n]) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was elected when the old leader resigned)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ main.step("Run for election on old leader(just so everyone is in the hat)")
+ run_result = old_leader.election_test_run()
+ if consistent_leader == main.TRUE:
+ after_run = main.ONOScli1.election_test_leader()
+ #verify leader didn't just change
+ if after_run == leader_list[0]:
+ leader_result = main.TRUE
+ else:
+ leader_result = main.FALSE
+ #TODO: assert on run and withdraw results?
+
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election after the old leader re-ran for election")
+
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index e1e75f9..78e0c83 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -1,9 +1,10 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
- <Git>True</Git>
+ <Git>False</Git>
+ <num_controllers> 7 </num_controllers>
<CTRL>
<ip1>10.128.30.11</ip1>
@@ -28,7 +29,7 @@
<port7>6633</port7>
</CTRL>
<TESTONUSER>admin</TESTONUSER>
- <TESTONIP>10.128.30.10</TESTONIP>
+ <TESTONIP>10.128.30.9</TESTONIP>
<PING>
<source1>h8</source1>
<source2>h9</source2>
@@ -52,8 +53,8 @@
<target10>10.0.0.27</target10>
</PING>
<timers>
- <LinkDiscovery>2</LinkDiscovery>
- <SwitchDiscovery>2</SwitchDiscovery>
+ <LinkDiscovery>.2</LinkDiscovery>
+ <SwitchDiscovery>.2</SwitchDiscovery>
</timers>
<MNtcpdump>
<intf>eth0</intf>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 614d514..7399925 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -17,6 +17,8 @@
CASE11: Switch down
CASE12: Switch up
CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
'''
class HATestSanity:
@@ -63,6 +65,7 @@
global ONOS6_port
global ONOS7_ip
global ONOS7_port
+ global num_controllers
ONOS1_ip = main.params['CTRL']['ip1']
ONOS1_port = main.params['CTRL']['port1']
@@ -78,6 +81,7 @@
ONOS6_port = main.params['CTRL']['port6']
ONOS7_ip = main.params['CTRL']['ip7']
ONOS7_port = main.params['CTRL']['port7']
+ num_controllers = int(main.params['num_controllers'])
main.step("Applying cell variable to environment")
@@ -145,8 +149,7 @@
main.step("Checking if ONOS is up yet")
- #TODO: Refactor
- # check bundle:list?
+ #TODO check bundle:list?
for i in range(2):
onos1_isup = main.ONOSbench.isup(ONOS1_ip)
if not onos1_isup:
@@ -173,7 +176,6 @@
and onos4_isup and onos5_isup and onos6_isup and onos7_isup
if onos_isup_result == main.TRUE:
break
- # TODO: if it becomes an issue, we can retry this step a few times
cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
@@ -214,31 +216,12 @@
import json
import re
-
- '''
- ONOS1_ip = main.params['CTRL']['ip1']
- ONOS1_port = main.params['CTRL']['port1']
- ONOS2_ip = main.params['CTRL']['ip2']
- ONOS2_port = main.params['CTRL']['port2']
- ONOS3_ip = main.params['CTRL']['ip3']
- ONOS3_port = main.params['CTRL']['port3']
- ONOS4_ip = main.params['CTRL']['ip4']
- ONOS4_port = main.params['CTRL']['port4']
- ONOS5_ip = main.params['CTRL']['ip5']
- ONOS5_port = main.params['CTRL']['port5']
- ONOS6_ip = main.params['CTRL']['ip6']
- ONOS6_port = main.params['CTRL']['port6']
- ONOS7_ip = main.params['CTRL']['ip7']
- ONOS7_port = main.params['CTRL']['port7']
- '''
-
-
main.log.report("Assigning switches to controllers")
main.case("Assigning Controllers")
main.step("Assign switches to controllers")
for i in range (1,29):
- main.Mininet1.assign_sw_controller(sw=str(i),count=7,
+ main.Mininet1.assign_sw_controller(sw=str(i),count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -270,8 +253,217 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
+ #Manually assign mastership to the controller we want
+ role_call = main.TRUE
+ role_check = main.TRUE
+
+ device_id = main.ONOScli1.get_device("1000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2800")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS1_ip)
+ if ONOS1_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("2000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS2_ip)
+ if ONOS2_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("5000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6000")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS3_ip)
+ if ONOS3_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3004")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS4_ip)
+ if ONOS4_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3008")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3009")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3010")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3011")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3012")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3013")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3014")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3015")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3016")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("3017")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS5_ip)
+ if ONOS5_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6007")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS6_ip)
+ if ONOS6_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6018")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6019")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6020")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6021")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6022")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6023")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6024")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6025")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6026")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ device_id = main.ONOScli1.get_device("6027")['id']
+ role_call = role_call and main.ONOScli1.device_role(device_id, ONOS7_ip)
+ if ONOS7_ip in main.ONOScli1.get_role(device_id)['master']:
+ role_check = role_check and main.TRUE
+ else:
+ role_check = role_check and main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_call,
+ onpass="Re-assigned switch mastership to designated controller",
+ onfail="Something wrong with device_role calls")
+
+ utilities.assert_equals(expect = main.TRUE,actual=role_check,
+ onpass="Switches were successfully reassigned to designated controller",
+ onfail="Switches were not successfully reassigned")
+ mastership_check = mastership_check and role_call and role_check
+ utilities.assert_equals(expect = main.TRUE,actual=mastership_check,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
def CASE3(self,main) :
@@ -288,6 +480,16 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+ main.ONOScli2.feature_install("onos-app-fwd")
+ main.ONOScli3.feature_install("onos-app-fwd")
+ main.ONOScli4.feature_install("onos-app-fwd")
+ main.ONOScli5.feature_install("onos-app-fwd")
+ main.ONOScli6.feature_install("onos-app-fwd")
+ main.ONOScli7.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -304,22 +506,30 @@
main.ONOScli5.feature_uninstall("onos-app-fwd")
main.ONOScli6.feature_uninstall("onos-app-fwd")
main.ONOScli7.feature_uninstall("onos-app-fwd")
+ #timeout for fwd flows
+ time.sleep(10)
main.step("Add host intents")
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
- #NOTE: get host can return None
- #TODO: handle this
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
- tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ #NOTE: get host can return None
+ if host1_id and host2_id:
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ else:
+ main.log.error("Error, get_host() failed")
+ tmp_result = main.FALSE
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -365,6 +575,25 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
ONOS3_mastership = main.ONOScli3.roles()
@@ -421,6 +650,7 @@
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
ONOS2_intents = main.ONOScli2.intents( json_format=True )
ONOS3_intents = main.ONOScli3.intents( json_format=True )
@@ -475,6 +705,7 @@
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
ONOS2_flows = main.ONOScli2.flows( json_format=True )
ONOS3_flows = main.ONOScli3.flows( json_format=True )
@@ -532,7 +763,7 @@
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
@@ -583,7 +814,6 @@
devices.append( main.ONOScli5.devices() )
devices.append( main.ONOScli6.devices() )
devices.append( main.ONOScli7.devices() )
- '''
hosts = []
hosts.append( main.ONOScli1.hosts() )
hosts.append( main.ONOScli2.hosts() )
@@ -592,7 +822,6 @@
hosts.append( main.ONOScli5.hosts() )
hosts.append( main.ONOScli6.hosts() )
hosts.append( main.ONOScli7.hosts() )
- '''
ports = []
ports.append( main.ONOScli1.ports() )
ports.append( main.ONOScli2.ports() )
@@ -609,13 +838,98 @@
links.append( main.ONOScli5.links() )
links.append( main.ONOScli6.links() )
links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
main.step("Comparing ONOS topology to MN")
devices_results = main.TRUE
ports_results = main.TRUE
links_results = main.TRUE
- for controller in range(7): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -644,14 +958,16 @@
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result\
+ and consistent_paths_result
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -676,6 +992,24 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ ONOS1_master_not_null = main.ONOScli1.roles_not_null()
+ ONOS2_master_not_null = main.ONOScli2.roles_not_null()
+ ONOS3_master_not_null = main.ONOScli3.roles_not_null()
+ ONOS4_master_not_null = main.ONOScli4.roles_not_null()
+ ONOS5_master_not_null = main.ONOScli5.roles_not_null()
+ ONOS6_master_not_null = main.ONOScli6.roles_not_null()
+ ONOS7_master_not_null = main.ONOScli7.roles_not_null()
+ roles_not_null = ONOS1_master_not_null and ONOS2_master_not_null and\
+ ONOS3_master_not_null and ONOS4_master_not_null and\
+ ONOS5_master_not_null and ONOS6_master_not_null and\
+ ONOS7_master_not_null
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
ONOS2_mastership = main.ONOScli2.roles()
@@ -707,7 +1041,6 @@
and ONOS1_mastership == ONOS5_mastership\
and ONOS1_mastership == ONOS6_mastership\
and ONOS1_mastership == ONOS7_mastership:
- #mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
main.log.report("Switch roles are consistent across all ONOS nodes")
else:
@@ -789,25 +1122,35 @@
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
else:
- main.log.warn("ONOS1 intents: ", json.dumps(json.loads(ONOS1_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS2 intents: ", json.dumps(json.loads(ONOS2_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS3 intents: ", json.dumps(json.loads(ONOS3_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS4 intents: ", json.dumps(json.loads(ONOS4_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS5 intents: ", json.dumps(json.loads(ONOS5_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS6 intents: ", json.dumps(json.loads(ONOS6_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
- main.log.warn("ONOS7 intents: ", json.dumps(json.loads(ONOS7_intents),
- sort_keys=True, indent=4, separators=(',', ': ')))
+ main.log.warn("ONOS1 intents: ")
+ print json.dumps(json.loads(ONOS1_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS2 intents: ")
+ print json.dumps(json.loads(ONOS2_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS3 intents: ")
+ print json.dumps(json.loads(ONOS3_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS4 intents: ")
+ print json.dumps(json.loads(ONOS4_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS5 intents: ")
+ print json.dumps(json.loads(ONOS5_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS6 intents: ")
+ print json.dumps(json.loads(ONOS6_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
+ main.log.warn("ONOS7 intents: ")
+ print json.dumps(json.loads(ONOS7_intents),
+ sort_keys=True, indent=4, separators=(',', ': '))
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost
main.step("Compare current intents with intents before the failure")
+ #NOTE: this requires case 5 to pass for intent_state to be set.
+ # maybe we should stop the test if that fails?
if intent_state == ONOS1_intents:
same_intents = main.TRUE
main.log.report("Intents are consistent with before failure")
@@ -826,12 +1169,12 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
@@ -847,7 +1190,7 @@
#NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range(8,18):
main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ Loss_In_Pings = main.Mininet2.checkForLoss("/tmp/ping.h"+str(i)) or Loss_In_Pings
if Loss_In_Pings == main.TRUE:
main.log.info("Loss in ping detected")
elif Loss_In_Pings == main.ERROR:
@@ -859,9 +1202,35 @@
onpass="No Loss of connectivity",
onfail="Loss of dataplane connectivity detected")
+ #Test of LeadershipElection
+ #NOTE: this only works for the sanity test. In case of failures, leader will likely change
+ leader = ONOS1_ip
+ leader_result = main.TRUE
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == leader:
+ #all is well
+ #NOTE: In failure scenario, this could be a new node, maybe check != ONOS1
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
- #TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+
+ result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings) and roles_not_null\
+ and leader_result
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -903,89 +1272,173 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
- print "cond 1:" + str(topo_result == main.FALSE)
- print "cond 2:" + str(elapsed < 120)
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+ clusters = []
+ clusters.append( main.ONOScli1.clusters() )
+ clusters.append( main.ONOScli2.clusters() )
+ clusters.append( main.ONOScli3.clusters() )
+ clusters.append( main.ONOScli4.clusters() )
+ clusters.append( main.ONOScli5.clusters() )
+ clusters.append( main.ONOScli6.clusters() )
+ clusters.append( main.ONOScli7.clusters() )
+ paths = []
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli1.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli2.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli3.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli4.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli5.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli6.topology() )
+ paths.append( temp_topo.get('paths', False) )
+ temp_topo = main.ONOSbench.get_topology( main.ONOScli7.topology() )
+ paths.append( temp_topo.get('paths', False) )
- for controller in range(7): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ elapsed = time.time() - start_time
+ cli_time = time.time() - cli_start
+ print "CLI time: " + str(cli_time)
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ for controller in range(num_controllers):
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = topo_result and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+
+ #Compare json objects for hosts, dataplane clusters and paths
+
+ #hosts
+ consistent_hosts_result = main.TRUE
+ for controller in range( len( hosts ) ):
+ if not "Error" in hosts[controller]:
+ if hosts[controller] == hosts[0]:
+ continue
+ else:#hosts not consistent
+ main.log.report("hosts from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ main.log.warn( repr( hosts[controller] ) )
+ consistent_hosts_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting ONOS hosts from ONOS" + str(controller + 1) )
+ consistent_hosts_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " hosts response: " + repr(hosts[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_hosts_result,
+ onpass="Hosts view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of hosts")
+
+ #Strongly connected clusters of devices
+ consistent_clusters_result = main.TRUE
+ for controller in range( len( clusters ) ):
+ if not "Error" in clusters[controller]:
+ if clusters[controller] == clusters[0]:
+ continue
+ else:#clusters not consistent
+ main.log.report("clusters from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_clusters_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting dataplane clusters from ONOS" + str(controller + 1) )
+ consistent_clusters_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " clusters response: " + repr(clusters[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_clusters_result,
+ onpass="Clusters view is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different views of clusters")
+ num_clusters = len(json.loads(clusters[0])) #there should always only be one cluster
+ utilities.assert_equals(expect = 1, actual = num_clusters,
+ onpass="ONOS shows 1 SCC",
+ onfail="ONOS shows "+str(num_clusters) +" SCCs")
+
+
+ #paths
+ consistent_paths_result = main.TRUE
+ for controller in range( len( paths ) ):
+ if not "Error" in paths[controller]:
+ if paths[controller] == paths[0]:
+ continue
+ else:#paths not consistent
+ main.log.report("paths from ONOS" + str(controller + 1) + " is inconsistent with ONOS1")
+ consistent_paths_result = main.FALSE
+
+ else:
+ main.log.report("Error in getting paths from ONOS" + str(controller + 1) )
+ consistent_paths_result = main.FALSE
+ main.log.warn("ONOS" + str(controller + 1) + " paths response: " + repr(paths[controller]) )
+ utilities.assert_equals(expect = main.TRUE,actual=consistent_paths_result,
+ onpass="Paths count is consistent across all ONOS nodes",
+ onfail="ONOS nodes have different counts of paths")
+
+
+ topo_result = devices_results and ports_results and links_results\
+ and consistent_hosts_result and consistent_clusters_result and consistent_paths_result
+
+ topo_result = topo_result and int(count <= 2)
+ note = "note it takes about "+str( int(cli_time) )+" seconds for the test to make all the cli calls to fetch the topology from each ONOS instance"
+ main.log.report("Very crass estimate for topology discovery/convergence("+ str(note) + "): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
@@ -1000,7 +1453,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Turn off a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1022,7 +1475,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Restore a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -1045,7 +1498,7 @@
#NOTE: You should probably run a topology check after this
import time
- switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Killing a switch to ensure it is discovered correctly"
main.log.report(description)
@@ -1054,13 +1507,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -1068,7 +1526,8 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -1080,7 +1539,7 @@
main.Mininet1.add_link('s28', 's3')
main.Mininet1.add_link('s28', 's6')
main.Mininet1.add_link('s28', 'h28')
- main.Mininet1.assign_sw_controller(sw="28",count=7,
+ main.Mininet1.assign_sw_controller(sw="28",count=num_controllers,
ip1=ONOS1_ip,port1=ONOS1_port,
ip2=ONOS2_ip,port2=ONOS2_port,
ip3=ONOS3_ip,port3=ONOS3_port,
@@ -1090,9 +1549,15 @@
ip7=ONOS7_ip,port7=ONOS7_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -1100,14 +1565,41 @@
'''
import os
import time
+ #printing colors to terminal
+ colors = {}
+ colors['cyan'] = '\033[96m'
+ colors['purple'] = '\033[95m'
+ colors['blue'] = '\033[94m'
+ colors['green'] = '\033[92m'
+ colors['yellow'] = '\033[93m'
+ colors['red'] = '\033[91m'
+ colors['end'] = '\033[0m'
description = "Test Cleanup"
main.log.report(description)
main.case(description)
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS1_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS2:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS2_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS3:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS3_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS4:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS4_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS5:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS5_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS6:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS6_ip)
+ print colors['purple'] + "Checking logs for errors on ONOS7:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS7_ip)
+
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -1119,19 +1611,26 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
#std*.log's
@@ -1141,24 +1640,29 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS2_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS2_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS2-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS3_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS3_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS3-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS4_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS4_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS4-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS5_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS5_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS5-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS6_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS6_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS6-"+f )
- main.ONOSbench.secureCopy( "sdn", ONOS7_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS7_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS7-"+f )
-
-
#sleep so scp can finish
time.sleep(10)
main.step("Packing and rotating pcap archives")
@@ -1169,3 +1673,139 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ leader_result = main.TRUE
+ #install app on onos 1
+ main.log.info("Install leadership election app")
+ main.ONOScli1.feature_install("onos-app-election")
+ #wait for election
+ #check for leader
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ #all is well
+ pass
+ elif leader == None:
+ #No leader elected
+ main.log.report("No leader was elected")
+ leader_result = main.FALSE
+ elif leader == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ else:
+ #error in response
+ main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+ leader_result = main.FALSE
+
+
+
+
+ #install on other nodes and check for leader.
+ #Should be onos1 and each app should show the same leader
+ for controller in range(2,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ node.feature_install("onos-app-election")
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == ONOS1_ip:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+ def CASE15 ( self, main ) :
+ '''
+ Check that Leadership Election is still functional
+ '''
+ leader_result = main.TRUE
+ description = "Check that Leadership Election is still functional"
+ main.log.report(description)
+ main.case(description)
+ main.step("Find current leader and withdraw")
+ leader = main.ONOScli1.election_test_leader()
+ #TODO: do some sanity checking on leader before using it
+ withdraw_result = main.FALSE
+ if leader == ONOS1_ip:
+ old_leader = getattr( main, "ONOScli1" )
+ elif leader == ONOS2_ip:
+ old_leader = getattr( main, "ONOScli2" )
+ elif leader == ONOS3_ip:
+ old_leader = getattr( main, "ONOScli3" )
+ elif leader == ONOS4_ip:
+ old_leader = getattr( main, "ONOScli4" )
+ elif leader == ONOS5_ip:
+ old_leader = getattr( main, "ONOScli5" )
+ elif leader == ONOS6_ip:
+ old_leader = getattr( main, "ONOScli6" )
+ elif leader == ONOS7_ip:
+ old_leader = getattr( main, "ONOScli7" )
+ elif leader == None or leader == main.FALSE:
+ main.log.report("Leader for the election app should be an ONOS node,"\
+ +"instead got '"+str(leader)+"'")
+ leader_result = main.FALSE
+ withdraw_result = old_leader.election_test_withdraw()
+
+
+ main.step("Make sure new leader is elected")
+ leader_list = []
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leader_list.append( node.election_test_leader() )
+ for leaderN in leader_list:
+ if leaderN == leader:
+ main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+ " as leader after they withdrew")
+ leader_result = main.FALSE
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ consistent_leader = main.FALSE
+ if len( set( leader_list ) ) == 1:
+ main.log.info("Each Election-app sees '"+str(leader_list[0])+"' as the leader")
+ consistent_leader = main.TRUE
+ else:
+ main.log.report("Inconsistent responses for leader of Election-app:")
+ for n in range(len(leader_list)):
+ main.log.report("ONOS" + str(n+1) + " response: " + str(leader_list[n]) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was elected when the old leader resigned)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ main.step("Run for election on old leader(just so everyone is in the hat)")
+ run_result = old_leader.election_test_run()
+ if consistent_leader == main.TRUE:
+ after_run = main.ONOScli1.election_test_leader()
+ #verify leader didn't just change
+ if after_run == leader_list[0]:
+ leader_result = main.TRUE
+ else:
+ leader_result = main.FALSE
+ #TODO: assert on run and withdraw results?
+
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election after the old leader re-ran for election")
+
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
index e1e75f9..95a75d7 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.params
@@ -1,9 +1,10 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
- <Git>True</Git>
+ <Git>False</Git>
+ <num_controllers> 1 </num_controllers>
<CTRL>
<ip1>10.128.30.11</ip1>
@@ -28,7 +29,7 @@
<port7>6633</port7>
</CTRL>
<TESTONUSER>admin</TESTONUSER>
- <TESTONIP>10.128.30.10</TESTONIP>
+ <TESTONIP>10.128.30.9</TESTONIP>
<PING>
<source1>h8</source1>
<source2>h9</source2>
@@ -52,8 +53,8 @@
<target10>10.0.0.27</target10>
</PING>
<timers>
- <LinkDiscovery>2</LinkDiscovery>
- <SwitchDiscovery>2</SwitchDiscovery>
+ <LinkDiscovery>.2</LinkDiscovery>
+ <SwitchDiscovery>.2</SwitchDiscovery>
</timers>
<MNtcpdump>
<intf>eth0</intf>
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
index e6f2fde..6a6fe23 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
@@ -16,6 +16,8 @@
CASE11: Switch down
CASE12: Switch up
CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
'''
class SingleInstanceHATestRestart:
@@ -61,6 +63,7 @@
global ONOS6_port
global ONOS7_ip
global ONOS7_port
+ global num_controllers
ONOS1_ip = main.params['CTRL']['ip1']
ONOS1_port = main.params['CTRL']['port1']
@@ -76,6 +79,7 @@
ONOS6_port = main.params['CTRL']['port6']
ONOS7_ip = main.params['CTRL']['ip7']
ONOS7_port = main.params['CTRL']['port7']
+ num_controllers = int(main.params['num_controllers'])
main.step("Applying cell variable to environment")
@@ -112,6 +116,8 @@
"clean install")
main.ONOSbench.get_version(report=True)
+ cell_result = main.ONOSbench.set_cell("SingleHA")
+ verify_result = main.ONOSbench.verify_cell()
main.step("Creating ONOS package")
package_result = main.ONOSbench.onos_package()
@@ -121,17 +127,18 @@
main.step("Checking if ONOS is up yet")
- #TODO: Refactor
- # check bundle:list?
- #this should be enough for ONOS to start
- time.sleep(60)
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ #TODO check bundle:list?
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if onos1_isup:
+ break
if not onos1_isup:
main.log.report("ONOS1 didn't start!")
+
# TODO: if it becomes an issue, we can retry this step a few times
- cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
+ cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
main.step("Start Packet Capture MN")
main.Mininet2.start_tcpdump(
@@ -142,7 +149,7 @@
case1_result = (clean_install_result and package_result and
cell_result and verify_result and onos1_install_result and
- onos1_isup and cli1_results)
+ onos1_isup and cli_result)
utilities.assert_equals(expect=main.TRUE, actual=case1_result,
onpass="Test startup successful",
@@ -186,8 +193,6 @@
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers")
- #TODO: If assign roles is working reliably then manually
- # assign mastership to the controller we want
def CASE3(self,main) :
@@ -195,6 +200,7 @@
Assign intents
"""
+ #FIXME: we must reinstall intents until we have a persistant datastore!
import time
import json
import re
@@ -204,6 +210,10 @@
main.step("Discovering Hosts( Via pingall for now)")
#FIXME: Once we have a host discovery mechanism, use that instead
+ #install onos-app-fwd
+ main.log.info("Install reactive forwarding app")
+ main.ONOScli1.feature_install("onos-app-fwd")
+
#REACTIVE FWD test
ping_result = main.FALSE
time1 = time.time()
@@ -214,22 +224,30 @@
#uninstall onos-app-fwd
main.log.info("Uninstall reactive forwarding app")
main.ONOScli1.feature_uninstall("onos-app-fwd")
+ #timeout for fwd flows
+ time.sleep(10)
main.step("Add host intents")
#TODO: move the host numbers to params
import json
intents_json= json.loads(main.ONOScli1.hosts())
- intent_add_result = main.FALSE
+ intent_add_result = True
for i in range(8,18):
main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10))
host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper()
host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper()
- #NOTE: get host can return None
- #TODO: handle this
host1_id = main.ONOScli1.get_host(host1)['id']
host2_id = main.ONOScli1.get_host(host2)['id']
- tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
- intent_add_result = intent_add_result and tmp_result
+ #NOTE: get host can return None
+ if host1_id and host2_id:
+ tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id )
+ else:
+ main.log.error("Error, get_host() failed")
+ tmp_result = main.FALSE
+ intent_add_result = bool(intent_add_result and tmp_result)
+ utilities.assert_equals(expect=True, actual=intent_add_result,
+ onpass="Switch mastership correctly assigned",
+ onfail="Error in (re)assigning switch mastership")
#TODO Check if intents all exist in datastore
#NOTE: Do we need to print this once the test is working?
#main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)),
@@ -275,6 +293,15 @@
main.step("Get the Mastership of each switch from each controller")
global mastership_state
+ mastership_state = []
+
+ #Assert that each device has a master
+ roles_not_null = main.ONOScli1.roles_not_null()
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
ONOS1_mastership = main.ONOScli1.roles()
#print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
#TODO: Make this a meaningful check
@@ -285,14 +312,11 @@
else:
mastership_state = ONOS1_mastership
consistent_mastership = main.TRUE
- main.log.report("Switch roles are consistent across all ONOS nodes")
- utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership,
- onpass="Switch roles are consistent across all ONOS nodes",
- onfail="ONOS nodes have different views of switch roles")
main.step("Get the intents from each controller")
global intent_state
+ intent_state = []
ONOS1_intents = main.ONOScli1.intents( json_format=True )
intent_check = main.FALSE
if "Error" in ONOS1_intents or not ONOS1_intents:
@@ -300,14 +324,11 @@
main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
else:
intent_check = main.TRUE
- main.log.report("Intents are consistent across all ONOS nodes")
- utilities.assert_equals(expect = main.TRUE,actual=intent_check,
- onpass="Intents are consistent across all ONOS nodes",
- onfail="ONOS nodes have different views of intents")
main.step("Get the flows from each controller")
global flow_state
+ flow_state = []
ONOS1_flows = main.ONOScli1.flows( json_format=True )
flow_check = main.FALSE
if "Error" in ONOS1_flows or not ONOS1_flows:
@@ -317,42 +338,16 @@
#TODO: Do a better check, maybe compare flows on switches?
flow_state = ONOS1_flows
flow_check = main.TRUE
- main.log.report("Flow count is consistent across all ONOS nodes")
- utilities.assert_equals(expect = main.TRUE,actual=flow_check,
- onpass="The flow count is consistent across all ONOS nodes",
- onfail="ONOS nodes have different flow counts")
main.step("Get the OF Table entries")
global flows
flows=[]
for i in range(1,29):
- flows.append(main.Mininet2.get_flowTable("s"+str(i),1.0))
+ flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i)))
#TODO: Compare switch flow tables with ONOS flow tables
- main.step("Start continuous pings")
- main.Mininet2.pingLong(src=main.params['PING']['source1'],
- target=main.params['PING']['target1'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source2'],
- target=main.params['PING']['target2'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source3'],
- target=main.params['PING']['target3'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source4'],
- target=main.params['PING']['target4'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source5'],
- target=main.params['PING']['target5'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source6'],
- target=main.params['PING']['target6'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source7'],
- target=main.params['PING']['target7'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source8'],
- target=main.params['PING']['target8'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source9'],
- target=main.params['PING']['target9'],pingTime=500)
- main.Mininet2.pingLong(src=main.params['PING']['source10'],
- target=main.params['PING']['target10'],pingTime=500)
-
main.step("Create TestONTopology object")
ctrls = []
count = 1
@@ -381,7 +376,7 @@
devices_results = main.TRUE
ports_results = main.TRUE
links_results = main.TRUE
- for controller in range(1): #TODO parameterize the number of controllers
+ for controller in range(num_controllers):
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
else:
@@ -417,7 +412,7 @@
final_assert = main.TRUE
final_assert = final_assert and topo_result and flow_check \
- and intent_check and consistent_mastership
+ and intent_check and consistent_mastership and roles_not_null
utilities.assert_equals(expect=main.TRUE, actual=final_assert,
onpass="State check successful",
onfail="State check NOT successful")
@@ -436,7 +431,7 @@
main.step("Checking if ONOS is up yet")
count = 0
- while count < 10
+ while count < 10:
onos1_isup = main.ONOSbench.isup(ONOS1_ip)
if onos1_isup == main.TRUE:
elapsed = time.time() - start
@@ -446,11 +441,12 @@
cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
- case_results = main.TRUE and onosi1_isup and cli_result
+ case_results = main.TRUE and onos1_isup and cli_result
utilities.assert_equals(expect=main.TRUE, actual=case_results,
onpass="ONOS restart successful",
onfail="ONOS restart NOT successful")
- main.log.info("ONOS took %s seconds to restart" % str(elapsed) )
+ main.log.info("ESTIMATE: ONOS took %s seconds to restart" % str(elapsed) )
+ time.sleep(5)
def CASE7(self,main) :
'''
@@ -460,6 +456,14 @@
import json
main.case("Running ONOS Constant State Tests")
+ #Assert that each device has a master
+ roles_not_null = main.ONOScli1.roles_not_null()
+ utilities.assert_equals(expect = main.TRUE,actual=roles_not_null,
+ onpass="Each device has a master",
+ onfail="Some devices don't have a master assigned")
+
+
+
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
#FIXME: Refactor this whole case for single instance
@@ -508,13 +512,14 @@
main.log.report("Error in getting ONOS intents")
main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents))
else:
- intent_state = ONOS1_intents
intent_check = main.TRUE
main.log.report("Intents are consistent across all ONOS nodes")
utilities.assert_equals(expect = main.TRUE,actual=intent_check,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents")
+ #NOTE: Hazelcast has no durability, so intents are lost
+ '''
main.step("Compare current intents with intents before the failure")
if intent_state == ONOS1_intents:
same_intents = main.TRUE
@@ -526,6 +531,7 @@
onpass="Intents are consistent with before failure",
onfail="The Intents changed during failure")
intent_check = intent_check and same_intents
+ '''
@@ -534,42 +540,48 @@
flows2=[]
for i in range(28):
main.log.info("Checking flow table on s" + str(i+1))
- tmp_flows = main.Mininet2.get_flowTable("s"+str(i+1),1.0)
+ tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1))
flows2.append(tmp_flows)
- Flow_Tables = Flow_Tables and main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows)
+ Flow_Tables = Flow_Tables and temp_result
if Flow_Tables == main.FALSE:
main.log.info("Differences in flow table for switch: "+str(i+1))
- break
if Flow_Tables == main.TRUE:
main.log.report("No changes were found in the flow tables")
utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables,
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables")
- main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
- #FIXME: This check is always failing. Investigate cause
- #NOTE: this may be something to do with file permsissions
- # or slight change in format
- main.Mininet2.pingKill(main.params['TESTONUSER'], main.params['TESTONIP'])
- Loss_In_Pings = main.FALSE
- #NOTE: checkForLoss returns main.FALSE with 0% packet loss
- for i in range(8,18):
- main.log.info("Checking for a loss in pings along flow from s" + str(i))
- Loss_In_Pings = Loss_In_Pings or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
- if Loss_In_Pings == main.TRUE:
- main.log.info("Loss in ping detected")
- elif Loss_In_Pings == main.ERROR:
- main.log.info("There are multiple mininet process running")
- elif Loss_In_Pings == main.FALSE:
- main.log.info("No Loss in the pings")
- main.log.report("No loss of dataplane connectivity")
- utilities.assert_equals(expect=main.FALSE,actual=Loss_In_Pings,
- onpass="No Loss of connectivity",
- onfail="Loss of dataplane connectivity detected")
+
+ #Test of LeadershipElection
+
+ leader = ONOS1_ip
+ leader_result = main.TRUE
+ for controller in range(1,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ #NOTE even though we restarted ONOS, it is the only one so onos 1 must be leader
+ if leaderN == leader:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a new leader was re-elected if applicable)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
- #TODO:add topology to this or leave as a seperate case?
- result = mastership_check and intent_check and Flow_Tables and (not Loss_In_Pings)
+ result = mastership_check and intent_check and Flow_Tables and roles_not_null\
+ and leader_result
result = int(result)
if result == main.TRUE:
main.log.report("Constant State Tests Passed")
@@ -606,62 +618,62 @@
ports_results = main.TRUE
links_results = main.TRUE
topo_result = main.FALSE
- start_time = time.time()
elapsed = 0
count = 0
- while topo_result == main.FALSE and elapsed < 120:
+ main.step("Collecting topology information from ONOS")
+ start_time = time.time()
+ while topo_result == main.FALSE and elapsed < 60:
count = count + 1
- try:
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- for controller in range(1): #TODO parameterize the number of controllers
- if devices[controller] or not "Error" in devices[controller]:
- current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
- else:
- current_devices_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
- onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
- onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
+ if count > 1:
+ MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
+ cli_start = time.time()
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ elapsed = time.time() - start_time
+ print "CLI time: " + str(time.time() - cli_start)
- if ports[controller] or not "Error" in ports[controller]:
- current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
- else:
- current_ports_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
- onpass="ONOS"+str(int(controller+1))+" ports view is correct",
- onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ for controller in range(num_controllers):
+ if devices[controller] or not "Error" in devices[controller]:
+ current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
+ else:
+ current_devices_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_devices_result,
+ onpass="ONOS"+str(int(controller+1))+" Switches view is correct",
+ onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect")
- if links[controller] or not "Error" in links[controller]:
- current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
- else:
- current_links_result = main.FALSE
- utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
- onpass="ONOS"+str(int(controller+1))+" links view is correct",
- onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
- except:
- main.log.error("something went wrong in topo comparison")
- main.log.warn( repr( devices ) )
- main.log.warn( repr( ports ) )
- main.log.warn( repr( links ) )
+ if ports[controller] or not "Error" in ports[controller]:
+ current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller]))
+ else:
+ current_ports_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_ports_result,
+ onpass="ONOS"+str(int(controller+1))+" ports view is correct",
+ onfail="ONOS"+str(int(controller+1))+" ports view is incorrect")
+ if links[controller] or not "Error" in links[controller]:
+ current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller]))
+ else:
+ current_links_result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE, actual=current_links_result,
+ onpass="ONOS"+str(int(controller+1))+" links view is correct",
+ onfail="ONOS"+str(int(controller+1))+" links view is incorrect")
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- elapsed = time.time() - start_time
- time_threshold = elapsed < 1
- topo_result = devices_results and ports_results and links_results and time_threshold
- #TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ topo_result = devices_results and ports_results and links_results
+
+ topo_result = topo_result and int(count <= 2)
+ main.log.report("Very crass estimate for topology discovery/convergence(note it takes about 1 seconds to read the topology from each ONOS instance): " +\
str(elapsed) + " seconds, " + str(count) +" tries" )
+ if elapsed > 60:
+ main.log.report("Giving up on topology convergence")
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
@@ -675,7 +687,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Turn off a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -697,7 +709,7 @@
'''
#NOTE: You should probably run a topology check after this
- link_sleep = int(main.params['timers']['LinkDiscovery'])
+ link_sleep = float(main.params['timers']['LinkDiscovery'])
description = "Restore a link to ensure that Link Discovery is working properly"
main.log.report(description)
@@ -720,7 +732,7 @@
#NOTE: You should probably run a topology check after this
import time
- switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Killing a switch to ensure it is discovered correctly"
main.log.report(description)
@@ -729,13 +741,18 @@
#TODO: Make this switch parameterizable
main.step("Kill s28 ")
main.log.report("Deleting s28")
- #FIXME: use new dynamic topo functions
main.Mininet1.del_switch("s28")
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
time.sleep(switch_sleep)
+ device = main.ONOScli1.get_device(dpid="0028")
#Peek at the deleted switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == False:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="Kill switch succesful",
+ onfail="Failed to kill switch?")
def CASE12 (self, main) :
'''
@@ -743,7 +760,8 @@
'''
#NOTE: You should probably run a topology check after this
import time
- #FIXME: use new dynamic topo functions
+
+ switch_sleep = float(main.params['timers']['SwitchDiscovery'])
description = "Adding a switch to ensure it is discovered correctly"
main.log.report(description)
main.case(description)
@@ -759,9 +777,15 @@
ip1=ONOS1_ip,port1=ONOS1_port)
main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered")
time.sleep(switch_sleep)
- #Peek at the added switch
- main.log.warn(main.ONOScli1.get_device(dpid="0028"))
- #TODO do some sort of check here
+ device = main.ONOScli1.get_device(dpid="0028")
+ #Peek at the deleted switch
+ main.log.warn( str(device) )
+ result = main.FALSE
+ if device and device['available'] == True:
+ result = main.TRUE
+ utilities.assert_equals(expect=main.TRUE,actual=result,
+ onpass="add switch succesful",
+ onfail="Failed to add switch?")
def CASE13 (self, main) :
'''
@@ -769,14 +793,28 @@
'''
import os
import time
+ #printing colors to terminal
+ colors = {}
+ colors['cyan'] = '\033[96m'
+ colors['purple'] = '\033[95m'
+ colors['blue'] = '\033[94m'
+ colors['green'] = '\033[92m'
+ colors['yellow'] = '\033[93m'
+ colors['red'] = '\033[91m'
+ colors['end'] = '\033[0m'
description = "Test Cleanup"
main.log.report(description)
main.case(description)
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
+ main.step("Checking ONOS Logs for errors")
+ print colors['purple'] + "Checking logs for errors on ONOS1:" + colors['end']
+ print main.ONOSbench.check_logs(ONOS1_ip)
main.step("Copying MN pcap and ONOS log files to test station")
testname = main.TEST
+ teststation_user = main.params['TESTONUSER']
+ teststation_IP = main.params['TESTONIP']
#NOTE: MN Pcap file is being saved to ~/packet_captures
# scp this file as MN and TestON aren't necessarily the same vm
#FIXME: scp
@@ -788,8 +826,11 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
+ main.ONOSbench.handle.expect("\$")
+ print main.ONOSbench.handle.before
#std*.log's
#NOTE: must end in /
@@ -798,7 +839,8 @@
#NOTE: must end in /
dst_dir = "~/packet_captures/"
for f in log_files:
- main.ONOSbench.secureCopy( "sdn", ONOS1_ip,log_folder+f,"rocks",\
+ main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+
+ teststation_user +"@"+teststation_IP+":"+\
dst_dir + str(testname) + "-ONOS1-"+f )
@@ -812,3 +854,117 @@
utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
onpass="Test cleanup successful",
onfail="Test cleanup NOT successful")
+
+ def CASE14 ( self, main ) :
+ '''
+ start election app on all onos nodes
+ '''
+ leader_result = main.TRUE
+ #install app on onos 1
+ main.log.info("Install leadership election app")
+ main.ONOScli1.feature_install("onos-app-election")
+ #wait for election
+ #check for leader
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ #all is well
+ pass
+ elif leader == None:
+ #No leader elected
+ main.log.report("No leader was elected")
+ leader_result = main.FALSE
+ elif leader == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ else:
+ #error in response
+ main.log.report("Unexpected response from election_test_leader function:'"+str(leader)+"'")
+ leader_result = main.FALSE
+
+
+
+
+ #install on other nodes and check for leader.
+ #Should be onos1 and each app should show the same leader
+ for controller in range(2,num_controllers+1):
+ node = getattr( main, ( 'ONOScli' + str( controller ) ) )#loop through ONOScli handlers
+ node.feature_install("onos-app-election")
+ leaderN = node.election_test_leader()
+ #verify leader is ONOS1
+ if leaderN == ONOS1_ip:
+ #all is well
+ pass
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leader != leaderN:
+ leader_result = main.FALSE
+ main.log.report("ONOS" + str(controller) + " sees "+str(leaderN) +
+ " as the leader of the election app. Leader should be "+str(leader) )
+ if leader_result:
+ main.log.report("Leadership election tests passed(consistent view of leader across listeners and a leader was elected)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+ def CASE15 ( self, main ) :
+ '''
+ Check that Leadership Election is still functional
+ '''
+ leader_result = main.TRUE
+ description = "Check that Leadership Election is still functional"
+ main.log.report(description)
+ main.case(description)
+ main.step("Find current leader and withdraw")
+ leader = main.ONOScli1.election_test_leader()
+ #TODO: do some sanity checking on leader before using it
+ withdraw_result = main.FALSE
+ if leader == ONOS1_ip:
+ old_leader = getattr( main, "ONOScli1" )
+ elif leader == None or leader == main.FALSE:
+ main.log.report("Leader for the election app should be an ONOS node,"\
+ +"instead got '"+str(leader)+"'")
+ leader_result = main.FALSE
+ withdraw_result = old_leader.election_test_withdraw()
+
+
+ main.step("Make sure new leader is elected")
+ leader_list = []
+ leaderN = main.ONOScli1.election_test_leader()
+ if leaderN == leader:
+ main.log.report("ONOS"+str(controller)+" still sees " + str(leader) +\
+ " as leader after they withdrew")
+ leader_result = main.FALSE
+ elif leaderN == main.FALSE:
+ #error in response
+ #TODO: add check for "Command not found:" in the driver, this means the app isn't loaded
+ main.log.report("Something is wrong with election_test_leader function, check the error logs")
+ leader_result = main.FALSE
+ elif leaderN == None:
+ main.log.info("There is no leader after the app withdrew from election")
+ if leader_result:
+ main.log.report("Leadership election tests passed(There is no leader after the old leader resigned)")
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="Something went wrong with Leadership election")
+
+
+ main.step("Run for election on old leader(just so everyone is in the hat)")
+ run_result = old_leader.election_test_run()
+ leader = main.ONOScli1.election_test_leader()
+ #verify leader is ONOS1
+ if leader == ONOS1_ip:
+ leader_result = main.TRUE
+ else:
+ leader_result = main.FALSE
+ #TODO: assert on run and withdraw results?
+
+ utilities.assert_equals(expect=main.TRUE, actual=leader_result,
+ onpass="Leadership election passed",
+ onfail="ONOS1's election app was not leader after it re-ran for election")
+