Committing the Basic HATests
diff --git a/TestON/tests/HATestONOS/HATestONOS.params b/TestON/tests/HATestONOS/HATestONOS.params
new file mode 100644
index 0000000..6952d1b
--- /dev/null
+++ b/TestON/tests/HATestONOS/HATestONOS.params
@@ -0,0 +1,50 @@
+<PARAMS>
+ <testcases>1,2,3,4,5,6,7,8</testcases>
+ <CTRL>
+ <ip1>10.128.9.1</ip1>
+ <port1>6633</port1>
+ <restPort1>8080</restPort1>
+ <ip2>10.128.9.2</ip2>
+ <port2>6633</port2>
+ <restPort2>8080</restPort2>
+ <ip3>10.128.9.3</ip3>
+ <port3>6633</port3>
+ <restPort3>8080</restPort3>
+ <ip4>10.128.9.4</ip4>
+ <port4>6633</port4>
+ <restPort4>8080</restPort4>
+ <ip5>10.128.9.5</ip5>
+ <port5>6633</port5>
+ <restPort5>8080</restPort5>
+ <switchURL>/wm/onos/registry/switches/json</switchURL>
+ <intentHighURL>/wm/onos/intent/high</intentHighURL>
+ <intentLowURL>/wm/onos/intent/low</intentLowURL>
+ </CTRL>
+ <TopoRest>/wm/onos/topology</TopoRest>
+ <INTENTS>
+ <intentPort>8080</intentPort>
+ <intentURL>wm/onos/intent</intentURL>
+ </INTENTS>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+</PARAMS>
diff --git a/TestON/tests/HATestONOS/HATestONOS.py b/TestON/tests/HATestONOS/HATestONOS.py
new file mode 100644
index 0000000..3511877
--- /dev/null
+++ b/TestON/tests/HATestONOS/HATestONOS.py
@@ -0,0 +1,458 @@
+
+class HATestONOS:
+
+ global topology
+ global masterSwitchList
+ global highIntentList
+ global lowIntentList
+ global flows
+ flows = []
+
+ def __init__(self) :
+ self.default = ''
+
+ '''
+ CASE1 is to close any existing instances of ONOS, clean out the
+ RAMCloud database, and start up ONOS instances.
+ '''
+ def CASE1(self,main) :
+ main.case("Initial Startup")
+ main.step("Stop ONOS")
+ main.ONOS1.stop_all()
+ main.ONOS2.stop_all()
+ main.ONOS3.stop_all()
+ main.ONOS4.stop_all()
+ main.ONOS5.stop_all()
+ main.ONOS1.stop_rest()
+ main.ONOS2.stop_rest()
+ main.ONOS3.stop_rest()
+ main.ONOS4.stop_rest()
+ main.ONOS5.stop_rest()
+ result = main.ONOS1.status() or main.ONOS2.status() \
+ or main.ONOS3.status() or main.ONOS4.status() or main.ONOS5.status()
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="ONOS stopped successfully",onfail="ONOS WAS NOT KILLED PROPERLY")
+ main.step("Startup Zookeeper")
+ main.ZK1.start()
+ main.ZK2.start()
+ main.ZK3.start()
+ main.ZK4.start()
+ main.ZK5.start()
+ result = main.ZK1.isup() and main.ZK2.isup()\
+ and main.ZK3.isup() and main.ZK4.isup() and main.ZK5.isup()
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Zookeeper started successfully",onfail="ZOOKEEPER FAILED TO START")
+ main.step("Cleaning RC Database and Starting All")
+ main.RC1.del_db()
+ main.RC2.del_db()
+ main.RC3.del_db()
+ main.RC4.del_db()
+ main.RC5.del_db()
+ main.ONOS1.start_all()
+ main.ONOS2.start_all()
+ main.ONOS3.start_all()
+ main.ONOS4.start_all()
+ main.ONOS5.start_all()
+ # main.ONOS1.start_rest()
+ main.step("Testing Startup")
+ result1 = main.ONOS1.rest_status()
+ vm1 = main.RC1.status_coor and main.RC1.status_serv and \
+ main.ONOS1.isup()
+ vm2 = main.RC2.status_coor and main.ONOS2.isup()
+ vm3 = main.RC3.status_coor and main.ONOS3.isup()
+ vm4 = main.RC4.status_coor and main.ONOS4.isup()
+ vm5 = main.RC5.status_coor and main.ONOS5.isup()
+ result = result1 and vm1 and vm2 and vm3 and vm4 and vm5
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Everything started successfully",onfail="EVERYTHING FAILED TO START")
+
+ '''
+ CASE2
+ '''
+ def CASE2(self,main) :
+ import time
+ import json
+ import re
+ main.log.report("Assigning Controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign Master Controllers")
+ for i in range(1,29):
+ if i ==1:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ elif i>=2 and i<5:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip2'],port1=main.params['CTRL']['port2'])
+ elif i>=5 and i<8:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip3'],port1=main.params['CTRL']['port3'])
+ elif i>=8 and i<18:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip4'],port1=main.params['CTRL']['port4'])
+ elif i>=18 and i<28:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip5'],port1=main.params['CTRL']['port5'])
+ else:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+
+ result = main.TRUE
+ for i in range (1,29):
+ if i==1:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=2 and i<5:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip2'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=5 and i<8:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip3'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=8 and i<18:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip4'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=18 and i<28:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip5'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ else:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is" + str(response))
+ if re.search("tcp:" +main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=result,onpass="MasterControllers assigned correctly")
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+
+ def CASE3(self,main) :
+ import time
+ import json
+ import re
+ main.case("Adding Intents")
+ intentIP = main.params['CTRL']['ip1']
+ intentPort=main.params['INTENTS']['intentPort']
+ intentURL=main.params['INTENTS']['intentURL']
+ count = 1
+ for i in range(8,18):
+ srcMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ dstMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ srcDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ dstDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ dstDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ srcDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ dstMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ srcMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ count = 1
+ i = 8
+ result = main.TRUE
+ while i <18 :
+ main.log.info("\n\nh"+str(i)+" is Pinging h" + str(i+10))
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ if ping ==main.FALSE and count <9:
+ count+=1
+ i = 8
+ result = main.FALSE
+ main.log.info("Ping FAILED! Making attempt number "+str(count) + "in 2 seconds")
+ time.sleep(2)
+ elif ping==main.FALSE:
+ main.log.info("PINGS FAILED! MAX RETRIES REACHED!")
+ i=19
+ result = main.FALSE
+ elif ping==main.TRUE:
+ main.log.info("Ping passed!")
+ i+=1
+ result = main.TRUE
+ else:
+ main.log.info("ERROR!!")
+ result = main.ERROR
+ if result==main.FALSE:
+ main.log.info("INTENTS HAVE NOT BEEN INSTALLED CORRECTLY!! EXITING!!!")
+ main.cleanup()
+ main.exit()
+
+
+ def CASE4(self,main) :
+ import time
+ from subprocess import Popen, PIPE
+ main.case("Setting up and Gathering data for current state")
+ main.step("Get the current In-Memory Topology on each ONOS Instance")
+
+ '''
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip'+str(count)) in main.params['CTRL']:
+ temp = temp+(getattr(main,('ONOS'+str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count+=1
+ else:
+ break
+ topo_result = main.TRUE
+
+ for n in range(1,count):
+ temp_result = main.Mininet1.compare_topo(ctrls,main.ONOS1.get_json(main.params['CTRL']['ip'+str(n)]+":"+main.params['CTRL']['restPort'+str(n)]+main.params['TopoRest']))
+ '''
+ '''
+ main.step("Get the Mastership of each switch")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ global masterSwitchList1
+ masterSwitchList1 = stdout
+ '''
+
+ main.step("Get the High Level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ global highIntentList1
+ highIntentList1 = stdout
+
+ main.step("Get the Low level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ global lowIntentList1
+ lowIntentList1= stdout
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,29):
+ flows.append(main.Mininet2.get_flowTable("s"+str(i)))
+
+
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+
+ def CASE5(self,main) :
+ import re
+ from random import randint
+ main.case("MAIN COMPONENT FAILURE AND SCENARIO SPECIFIC TESTS")
+ main.step("ONOS CORE Failure!")
+ kill = randint(1,4)
+ if kill==1:
+ main.ONOS2.stop()
+ elif kill==2:
+ main.ONOS2.stop()
+ elif kill==3:
+ main.ONOS3.stop()
+ elif kill==4:
+ main.ONOS4.stop()
+ else:
+ main.ONOS5.stop()
+
+ kill2 = randint(1,5)
+ if kill2==kill:
+ if kill2==5:
+ main.ONOS4.stop()
+ elif kill2==1:
+ main.ONOS3.stop()
+ else:
+ main.ONOS5.stop()
+ else:
+ if kill2==1:
+ main.ONOS5.stop()
+ elif kill2==2:
+ main.ONOS2.stop()
+ elif kill2==3:
+ main.ONOS3.stop()
+ elif kill2==4:
+ main.ONOS4.stop()
+ else:
+ main.ONOS5.stop()
+ time.sleep(10)
+ main.step("Get the Mastership of each switch")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ global masterSwitchList1
+ masterSwitchList1 = stdout
+
+
+
+ def CASE6(self,main) :
+ import os
+ main.case("Running ONOS Constant State Tests")
+ main.step("Get the current In-Memory Topology on each ONOS Instance and Compare it to the Topology before component failure")
+
+ main.step("Get the Mastership of each switch and compare to the Mastership before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ result = main.TRUE
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+ switchDPID = switchDPID[:2]+":"+switchDPID[2:4]+":"+switchDPID[4:6]+":"+switchDPID[6:8]+":"+switchDPID[8:10]+":"+switchDPID[10:12]+":"+switchDPID[12:14]+":"+switchDPID[14:]
+ master1 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)
+ master2 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout)
+ if main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)==main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Mastership of Switches was not changed",onfail="MASTERSHIP OF SWITCHES HAS CHANGED!!!")
+ result1 = result
+
+ main.step("Get the High Level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_intents(preIntents=highIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE HIGH LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to High level Intents",onfail="CHANGES WERE MADE TO HIGH LEVEL INTENTS")
+ result2 = result
+
+ main.step("Get the Low level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_low(preIntents=lowIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE LOW LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to Low level Intents",onfail="CHANGES WERE MADE TO LOW LEVEL INTENTS")
+ result3 = result
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ result = main.TRUE
+ flows2=[]
+ for i in range(27):
+ flows2.append(main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ result = result and main.Mininet2.flow_comp(flow1=flows[i], flow2=main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ if result == main.FALSE:
+ main.log.info("DIFFERENCES IN FLOW TABLES FOR SWITCH "+str(i))
+ break
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes in the flow tables",onfail="CHANGES IN THE FLOW TABLES!!")
+ result4 = result
+
+ main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result5 = not result
+ result = result1 and result2 and result3 and result4 and result5
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Constant State Tests Passed", onfail="CONSTANT STATE TESTS FAILED!!")
+
+ def CASE7 (self,main):
+ main.case("Killing a link to Ensure that Link Discovery is Working Properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill Link between s3 and s28")
+ main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ time.sleep(5)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,str(int(links)-2))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link Down discovered properly",onfail="LINKS NOT DISCOVERED PROPERLY")
+ result1 = result
+ result = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+
+ main.step("Check for loss in pings when Link is brought down")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result2 = result
+ result = result1 and not result2
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link failure is discovered correctly and no traffic is lost!",onfail="Link Discovery failed or traffic was dropped!!!")
+
+ def CASE8 (self, main) :
+ import time
+ main.case("Killing a switch to ensure switch discovery is working properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill s28 ")
+ main.Mininet2.del_switch("s28")
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],str(int(active)-1),str(int(links)-4))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+
+ main.step("Add back s28")
+ main.Mininet2.add_switch("s28")
+ main.Mininet1.assign_sw_controller(sw="28",ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ main.Mininet1.assign_sw_controller(sw="28",count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,links)
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+ result1=result
+
+ main.step("Checking for Traffic Loss")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result = not result and result1
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovered Correctly and No Loss of traffic",onfail="Switch discovery failed or there was loss of traffic")
diff --git a/TestON/tests/HATestONOS/HATestONOS.topo b/TestON/tests/HATestONOS/HATestONOS.topo
new file mode 100644
index 0000000..9ccdb17
--- /dev/null
+++ b/TestON/tests/HATestONOS/HATestONOS.topo
@@ -0,0 +1,183 @@
+<TOPOLOGY>
+
+ <COMPONENT>
+ <ZK1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK1>
+
+ <ZK2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK2>
+
+ <ZK3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK3>
+
+ <ZK4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK4>
+
+ <ZK5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK5>
+
+ <RC1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC1>
+
+ <RC2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC2>
+
+ <RC3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC3>
+
+ <RC4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>9</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC4>
+
+ <RC5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC5>
+
+ <ONOS1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>11</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>12</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS3>
+
+ <ONOS4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS4>
+
+ <ONOS5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>15</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS5>
+
+ <Mininet1>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>MininetCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet1>
+
+ <Mininet2>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>RemoteMininetDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-onos4node.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet2>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestONOS2/HATestONOS2.params b/TestON/tests/HATestONOS2/HATestONOS2.params
new file mode 100644
index 0000000..6952d1b
--- /dev/null
+++ b/TestON/tests/HATestONOS2/HATestONOS2.params
@@ -0,0 +1,50 @@
+<PARAMS>
+ <testcases>1,2,3,4,5,6,7,8</testcases>
+ <CTRL>
+ <ip1>10.128.9.1</ip1>
+ <port1>6633</port1>
+ <restPort1>8080</restPort1>
+ <ip2>10.128.9.2</ip2>
+ <port2>6633</port2>
+ <restPort2>8080</restPort2>
+ <ip3>10.128.9.3</ip3>
+ <port3>6633</port3>
+ <restPort3>8080</restPort3>
+ <ip4>10.128.9.4</ip4>
+ <port4>6633</port4>
+ <restPort4>8080</restPort4>
+ <ip5>10.128.9.5</ip5>
+ <port5>6633</port5>
+ <restPort5>8080</restPort5>
+ <switchURL>/wm/onos/registry/switches/json</switchURL>
+ <intentHighURL>/wm/onos/intent/high</intentHighURL>
+ <intentLowURL>/wm/onos/intent/low</intentLowURL>
+ </CTRL>
+ <TopoRest>/wm/onos/topology</TopoRest>
+ <INTENTS>
+ <intentPort>8080</intentPort>
+ <intentURL>wm/onos/intent</intentURL>
+ </INTENTS>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+</PARAMS>
diff --git a/TestON/tests/HATestONOS2/HATestONOS2.py b/TestON/tests/HATestONOS2/HATestONOS2.py
new file mode 100644
index 0000000..861d282
--- /dev/null
+++ b/TestON/tests/HATestONOS2/HATestONOS2.py
@@ -0,0 +1,429 @@
+
+class HATestONOS2:
+
+ global topology
+ global masterSwitchList
+ global highIntentList
+ global lowIntentList
+ global flows
+ flows = []
+
+ def __init__(self) :
+ self.default = ''
+
+ '''
+ CASE1 is to close any existing instances of ONOS, clean out the
+ RAMCloud database, and start up ONOS instances.
+ '''
+ def CASE1(self,main) :
+ main.case("Initial Startup")
+ main.step("Stop ONOS")
+ main.ONOS1.stop_all()
+ main.ONOS2.stop_all()
+ main.ONOS3.stop_all()
+ main.ONOS4.stop_all()
+ main.ONOS5.stop_all()
+ main.ONOS1.stop_rest()
+ main.ONOS2.stop_rest()
+ main.ONOS3.stop_rest()
+ main.ONOS4.stop_rest()
+ main.ONOS5.stop_rest()
+ result = main.ONOS1.status() or main.ONOS2.status() \
+ or main.ONOS3.status() or main.ONOS4.status() or main.ONOS5.status()
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="ONOS stopped successfully",onfail="ONOS WAS NOT KILLED PROPERLY")
+ main.step("Startup Zookeeper")
+ main.ZK1.start()
+ main.ZK2.start()
+ main.ZK3.start()
+ main.ZK4.start()
+ main.ZK5.start()
+ result = main.ZK1.isup() and main.ZK2.isup()\
+ and main.ZK3.isup() and main.ZK4.isup() and main.ZK5.isup()
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Zookeeper started successfully",onfail="ZOOKEEPER FAILED TO START")
+ main.step("Cleaning RC Database and Starting All")
+ main.RC1.del_db()
+ main.RC2.del_db()
+ main.RC3.del_db()
+ main.RC4.del_db()
+ main.RC5.del_db()
+ main.ONOS1.start_all()
+ main.ONOS2.start_all()
+ main.ONOS3.start_all()
+ main.ONOS4.start_all()
+ main.ONOS5.start_all()
+ # main.ONOS1.start_rest()
+ main.step("Testing Startup")
+ result1 = main.ONOS1.rest_status()
+ vm1 = main.RC1.status_coor and main.RC1.status_serv and \
+ main.ONOS1.isup()
+ vm2 = main.RC2.status_coor and main.ONOS2.isup()
+ vm3 = main.RC3.status_coor and main.ONOS3.isup()
+ vm4 = main.RC4.status_coor and main.ONOS4.isup()
+ vm5 = main.RC5.status_coor and main.ONOS5.isup()
+ result = result1 and vm1 and vm2 and vm3 and vm4 and vm5
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Everything started successfully",onfail="EVERYTHING FAILED TO START")
+
+ '''
+ CASE2
+ '''
+ def CASE2(self,main) :
+ import time
+ import json
+ import re
+ main.log.report("Assigning Controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign Master Controllers")
+ for i in range(1,29):
+ if i ==1:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ elif i>=2 and i<5:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip2'],port1=main.params['CTRL']['port2'])
+ elif i>=5 and i<8:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip3'],port1=main.params['CTRL']['port3'])
+ elif i>=8 and i<18:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip4'],port1=main.params['CTRL']['port4'])
+ elif i>=18 and i<28:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip5'],port1=main.params['CTRL']['port5'])
+ else:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+
+ result = main.TRUE
+ for i in range (1,29):
+ if i==1:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=2 and i<5:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip2'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=5 and i<8:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip3'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=8 and i<18:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip4'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=18 and i<28:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip5'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ else:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is" + str(response))
+ if re.search("tcp:" +main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=result,onpass="MasterControllers assigned correctly")
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+
+ def CASE3(self,main) :
+ import time
+ import json
+ import re
+ main.case("Adding Intents")
+ intentIP = main.params['CTRL']['ip1']
+ intentPort=main.params['INTENTS']['intentPort']
+ intentURL=main.params['INTENTS']['intentURL']
+ count = 1
+ for i in range(8,18):
+ srcMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ dstMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ srcDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ dstDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ dstDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ srcDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ dstMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ srcMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ count = 1
+ i = 8
+ result = main.TRUE
+ while i <18 :
+ main.log.info("\n\nh"+str(i)+" is Pinging h" + str(i+10))
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ if ping ==main.FALSE and count <9:
+ count+=1
+ i = 8
+ result = main.FALSE
+ main.log.info("Ping FAILED! Making attempt number "+str(count) + "in 2 seconds")
+ time.sleep(2)
+ elif ping==main.FALSE:
+ main.log.info("PINGS FAILED! MAX RETRIES REACHED!")
+ i=19
+ result = main.FALSE
+ elif ping==main.TRUE:
+ main.log.info("Ping passed!")
+ i+=1
+ result = main.TRUE
+ else:
+ main.log.info("ERROR!!")
+ result = main.ERROR
+ if result==main.FALSE:
+ main.log.info("INTENTS HAVE NOT BEEN INSTALLED CORRECTLY!! EXITING!!!")
+ main.cleanup()
+ main.exit()
+
+
+ def CASE4(self,main) :
+ import time
+ from subprocess import Popen, PIPE
+ main.case("Setting up and Gathering data for current state")
+ main.step("Get the current In-Memory Topology on each ONOS Instance")
+
+ '''
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip'+str(count)) in main.params['CTRL']:
+ temp = temp+(getattr(main,('ONOS'+str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count+=1
+ else:
+ break
+ topo_result = main.TRUE
+
+ for n in range(1,count):
+ temp_result = main.Mininet1.compare_topo(ctrls,main.ONOS1.get_json(main.params['CTRL']['ip'+str(n)]+":"+main.params['CTRL']['restPort'+str(n)]+main.params['TopoRest']))
+ '''
+ main.step("Get the Mastership of each switch")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ global masterSwitchList1
+ masterSwitchList1 = stdout
+
+ main.step("Get the High Level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ global highIntentList1
+ highIntentList1 = stdout
+
+ main.step("Get the Low level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ global lowIntentList1
+ lowIntentList1= stdout
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,29):
+ flows.append(main.Mininet2.get_flowTable("s"+str(i)))
+
+
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+
+ def CASE5(self,main) :
+ import re
+ from random import randint
+ main.case("MAIN COMPONENT FAILURE AND SCENARIO SPECIFIC TESTS")
+ main.step("ONOS CORE All - Failure!")
+ main.ONOS1.stop()
+ main.ONOS2.stop()
+ main.ONOS3.stop()
+ main.ONOS4.stop()
+ main.ONOS5.stop()
+ time.sleep(2)
+ main.ONOS1.start()
+ main.ONOS2.start()
+ main.ONOS3.start()
+ main.ONOS4.start()
+ main.ONOS5.start()
+ if main.ONOS1.isup() and main.ONOS2.isup() and main.ONOS3.isup() and main.ONOS4.isup() and main.ONOS5.isup():
+ main.log.info("ONOS BACK UP!")
+ result = main.TRUE
+ else:
+ main.log.info("ONOS DID NOT COME BACK UP!!!")
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="ONOS has been brought back up!",onfail = "ONOS HAS FAILED TO RESTART!")
+
+ def CASE6(self,main) :
+ import os
+ main.case("Running ONOS Constant State Tests")
+ main.step("Get the current In-Memory Topology on each ONOS Instance and Compare it to the Topology before component failure")
+
+ main.step("Get the Mastership of each switch and compare to the Mastership before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ result = main.TRUE
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+ switchDPID = switchDPID[:2]+":"+switchDPID[2:4]+":"+switchDPID[4:6]+":"+switchDPID[6:8]+":"+switchDPID[8:10]+":"+switchDPID[10:12]+":"+switchDPID[12:14]+":"+switchDPID[14:]
+ master1 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)
+ master2 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout)
+ if main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)==main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Mastership of Switches was not changed",onfail="MASTERSHIP OF SWITCHES HAS CHANGED!!!")
+
+ main.step("Get the High Level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_intents(preIntents=highIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE HIGH LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to High level Intents",onfail="CHANGES WERE MADE TO HIGH LEVEL INTENTS")
+
+ main.step("Get the Low level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_low(preIntents=lowIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE LOW LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to Low level Intents",onfail="CHANGES WERE MADE TO LOW LEVEL INTENTS")
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ result = main.TRUE
+ flows2=[]
+ for i in range(27):
+ flows2.append(main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ result = result and main.Mininet2.flow_comp(flow1=flows[i], flow2=main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ if result == main.FALSE:
+ main.log.info("DIFFERENCES IN FLOW TABLES FOR SWITCH "+str(i))
+ break
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes in the flow tables",onfail="CHANGES IN THE FLOW TABLES!!")
+
+ main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+
+ def CASE7 (self,main):
+ main.case("Killing a link to Ensure that Link Discovery is Working Properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill Link between s3 and s28")
+ main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ time.sleep(5)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,str(int(links)-2))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link Down discovered properly",onfail="LINKS NOT DISCOVERED PROPERLY")
+ result1 = result
+ result = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+
+ main.step("Check for loss in pings when Link is brought down")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result2 = result
+ result = result1 and not result2
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link failure is discovered correctly and no traffic is lost!",onfail="Link Discovery failed or traffic was dropped!!!")
+
+ def CASE8 (self, main) :
+ import time
+ main.case("Killing a switch to ensure switch discovery is working properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill s28 ")
+ main.Mininet2.del_switch("s28")
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],str(int(active)-1),str(int(links)-4))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+
+ main.step("Add back s28")
+ main.Mininet2.add_switch("s28")
+ main.Mininet1.assign_sw_controller(sw="28",ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ main.Mininet1.assign_sw_controller(sw="28",count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,links)
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+ result1=result
+
+ main.step("Checking for Traffic Loss")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result = not result and result1
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovered Correctly and No Loss of traffic",onfail="Switch discovery failed or there was loss of traffic")
diff --git a/TestON/tests/HATestONOS2/HATestONOS2.topo b/TestON/tests/HATestONOS2/HATestONOS2.topo
new file mode 100644
index 0000000..9ccdb17
--- /dev/null
+++ b/TestON/tests/HATestONOS2/HATestONOS2.topo
@@ -0,0 +1,183 @@
+<TOPOLOGY>
+
+ <COMPONENT>
+ <ZK1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK1>
+
+ <ZK2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK2>
+
+ <ZK3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK3>
+
+ <ZK4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK4>
+
+ <ZK5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK5>
+
+ <RC1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC1>
+
+ <RC2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC2>
+
+ <RC3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC3>
+
+ <RC4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>9</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC4>
+
+ <RC5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC5>
+
+ <ONOS1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>11</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>12</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS3>
+
+ <ONOS4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS4>
+
+ <ONOS5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>15</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS5>
+
+ <Mininet1>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>MininetCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet1>
+
+ <Mininet2>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>RemoteMininetDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-onos4node.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet2>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestRCS/HATestRCS.params b/TestON/tests/HATestRCS/HATestRCS.params
new file mode 100644
index 0000000..6952d1b
--- /dev/null
+++ b/TestON/tests/HATestRCS/HATestRCS.params
@@ -0,0 +1,50 @@
+<PARAMS>
+ <testcases>1,2,3,4,5,6,7,8</testcases>
+ <CTRL>
+ <ip1>10.128.9.1</ip1>
+ <port1>6633</port1>
+ <restPort1>8080</restPort1>
+ <ip2>10.128.9.2</ip2>
+ <port2>6633</port2>
+ <restPort2>8080</restPort2>
+ <ip3>10.128.9.3</ip3>
+ <port3>6633</port3>
+ <restPort3>8080</restPort3>
+ <ip4>10.128.9.4</ip4>
+ <port4>6633</port4>
+ <restPort4>8080</restPort4>
+ <ip5>10.128.9.5</ip5>
+ <port5>6633</port5>
+ <restPort5>8080</restPort5>
+ <switchURL>/wm/onos/registry/switches/json</switchURL>
+ <intentHighURL>/wm/onos/intent/high</intentHighURL>
+ <intentLowURL>/wm/onos/intent/low</intentLowURL>
+ </CTRL>
+ <TopoRest>/wm/onos/topology</TopoRest>
+ <INTENTS>
+ <intentPort>8080</intentPort>
+ <intentURL>wm/onos/intent</intentURL>
+ </INTENTS>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+</PARAMS>
diff --git a/TestON/tests/HATestRCS/HATestRCS.py b/TestON/tests/HATestRCS/HATestRCS.py
new file mode 100644
index 0000000..80413f8
--- /dev/null
+++ b/TestON/tests/HATestRCS/HATestRCS.py
@@ -0,0 +1,446 @@
+
+class HATestRCS:
+
+ global topology
+ global masterSwitchList
+ global highIntentList
+ global lowIntentList
+ global flows
+ flows = []
+
+ def __init__(self) :
+ self.default = ''
+
+ '''
+ CASE1 is to close any existing instances of ONOS, clean out the
+ RAMCloud database, and start up ONOS instances.
+ '''
+ def CASE1(self,main) :
+
+ main.case("Initial Startup")
+ main.step("Stop ONOS")
+ main.ONOS1.stop_all()
+ main.ONOS2.stop_all()
+ main.ONOS3.stop_all()
+ main.ONOS4.stop_all()
+ main.ONOS5.stop_all()
+ main.ONOS1.stop_rest()
+ main.ONOS2.stop_rest()
+ main.ONOS3.stop_rest()
+ main.ONOS4.stop_rest()
+ main.ONOS5.stop_rest()
+ result = main.ONOS1.status() or main.ONOS2.status() \
+ or main.ONOS3.status() or main.ONOS4.status() or main.ONOS5.status()
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="ONOS stopped successfully",onfail="ONOS WAS NOT KILLED PROPERLY")
+ main.step("Startup Zookeeper")
+ main.ZK1.start()
+ main.ZK2.start()
+ main.ZK3.start()
+ main.ZK4.start()
+ main.ZK5.start()
+ result = main.ZK1.isup() and main.ZK2.isup()\
+ and main.ZK3.isup() and main.ZK4.isup() and main.ZK5.isup()
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Zookeeper started successfully",onfail="ZOOKEEPER FAILED TO START")
+ main.step("Cleaning RC Database and Starting All")
+ main.RC1.del_db()
+ main.RC2.del_db()
+ main.RC3.del_db()
+ main.RC4.del_db()
+ main.RC5.del_db()
+ main.ONOS1.start_all()
+ main.ONOS2.start_all()
+ main.ONOS3.start_all()
+ main.ONOS4.start_all()
+ main.ONOS5.start_all()
+ # main.ONOS1.start_rest()
+ main.step("Testing Startup")
+ result1 = main.ONOS1.rest_status()
+ vm1 = main.RC1.status_coor and main.RC1.status_serv and \
+ main.ONOS1.isup()
+ vm2 = main.RC2.status_coor and main.ONOS2.isup()
+ vm3 = main.RC3.status_coor and main.ONOS3.isup()
+ vm4 = main.RC4.status_coor and main.ONOS4.isup()
+ vm5 = main.RC5.status_coor and main.ONOS5.isup()
+ result = result1 and vm1 and vm2 and vm3 and vm4 and vm5
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Everything started successfully",onfail="EVERYTHING FAILED TO START")
+
+ '''
+ CASE2
+ '''
+ def CASE2(self,main) :
+ import time
+ import json
+ import re
+ main.log.report("Assigning Controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign Master Controllers")
+ for i in range(1,29):
+ if i ==1:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ elif i>=2 and i<5:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip2'],port1=main.params['CTRL']['port2'])
+ elif i>=5 and i<8:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip3'],port1=main.params['CTRL']['port3'])
+ elif i>=8 and i<18:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip4'],port1=main.params['CTRL']['port4'])
+ elif i>=18 and i<28:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip5'],port1=main.params['CTRL']['port5'])
+ else:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+
+ result = main.TRUE
+ for i in range (1,29):
+ if i==1:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=2 and i<5:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip2'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=5 and i<8:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip3'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=8 and i<18:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip4'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=18 and i<28:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip5'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ else:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is" + str(response))
+ if re.search("tcp:" +main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=result,onpass="MasterControllers assigned correctly")
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+
+ def CASE3(self,main) :
+ import time
+ import json
+ import re
+ main.case("Adding Intents")
+ intentIP = main.params['CTRL']['ip1']
+ intentPort=main.params['INTENTS']['intentPort']
+ intentURL=main.params['INTENTS']['intentURL']
+ count = 1
+ for i in range(8,18):
+ srcMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ dstMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ srcDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ dstDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ dstDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ srcDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ dstMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ srcMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ count = 1
+ i = 8
+ result = main.TRUE
+ while i <18 :
+ main.log.info("\n\nh"+str(i)+" is Pinging h" + str(i+10))
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ if ping ==main.FALSE and count <9:
+ count+=1
+ i = 8
+ result = main.FALSE
+ main.log.info("Ping FAILED! Making attempt number "+str(count) + "in 2 seconds")
+ time.sleep(2)
+ elif ping==main.FALSE:
+ main.log.info("PINGS FAILED! MAX RETRIES REACHED!")
+ i=19
+ result = main.FALSE
+ elif ping==main.TRUE:
+ main.log.info("Ping passed!")
+ i+=1
+ result = main.TRUE
+ else:
+ main.log.info("ERROR!!")
+ result = main.ERROR
+ if result==main.FALSE:
+ main.log.info("INTENTS HAVE NOT BEEN INSTALLED CORRECTLY!! EXITING!!!")
+ main.cleanup()
+ main.exit()
+
+
+ def CASE4(self,main) :
+ import time
+ from subprocess import Popen, PIPE
+ main.case("Setting up and Gathering data for current state")
+ main.step("Get the current In-Memory Topology on each ONOS Instance")
+
+ '''
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip'+str(count)) in main.params['CTRL']:
+ temp = temp+(getattr(main,('ONOS'+str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count+=1
+ else:
+ break
+ topo_result = main.TRUE
+
+ for n in range(1,count):
+ temp_result = main.Mininet1.compare_topo(ctrls,main.ONOS1.get_json(main.params['CTRL']['ip'+str(n)]+":"+main.params['CTRL']['restPort'+str(n)]+main.params['TopoRest']))
+ '''
+ main.step("Get the Mastership of each switch")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ global masterSwitchList1
+ masterSwitchList1 = stdout
+
+ main.step("Get the High Level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ global highIntentList1
+ highIntentList1 = stdout
+
+ main.step("Get the Low level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ global lowIntentList1
+ lowIntentList1= stdout
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,29):
+ flows.append(main.Mininet2.get_flowTable("s"+str(i)))
+
+
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+
+ def CASE5(self,main) :
+ import re
+ from random import randint
+ main.case("MAIN COMPONENT FAILURE AND SCENARIO SPECIFIC TESTS")
+ main.step("RC CORE Failure!")
+ kill = randint(1,4)
+ if kill==1:
+ main.RC1.stop_serv()
+ elif kill==2:
+ main.RC2.stop_serv()
+ elif kill==3:
+ main.RC3.stop_serv()
+ elif kill==4:
+ main.RC4.stop_serv()
+ else:
+ main.RC5.stop_serv()
+
+ kill2 = randint(1,5)
+ if kill2==kill:
+ if kill2==5:
+ main.RC4.stop_serv()
+ elif kill2==1:
+ main.RC3.stop_serv()
+ else:
+ main.RC5.stop_serv()
+ else:
+ if kill2==1:
+ main.RC1.stop_serv()
+ elif kill2==2:
+ main.RC2.stop_serv()
+ elif kill2==3:
+ main.RC3.stop_serv()
+ elif kill2==4:
+ main.RC4.stop_serv()
+ else:
+ main.RC5.stop_serv()
+ time.sleep(10)
+
+
+
+ def CASE6(self,main) :
+ import os
+ main.case("Running ONOS Constant State Tests")
+ main.step("Get the current In-Memory Topology on each ONOS Instance and Compare it to the Topology before component failure")
+
+ main.step("Get the Mastership of each switch and compare to the Mastership before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ result = main.TRUE
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+ switchDPID = switchDPID[:2]+":"+switchDPID[2:4]+":"+switchDPID[4:6]+":"+switchDPID[6:8]+":"+switchDPID[8:10]+":"+switchDPID[10:12]+":"+switchDPID[12:14]+":"+switchDPID[14:]
+ master1 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)
+ master2 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout)
+ if main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)==main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Mastership of Switches was not changed",onfail="MASTERSHIP OF SWITCHES HAS CHANGED!!!")
+
+ main.step("Get the High Level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_intents(preIntents=highIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE HIGH LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to High level Intents",onfail="CHANGES WERE MADE TO HIGH LEVEL INTENTS")
+
+ main.step("Get the Low level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_low(preIntents=lowIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE LOW LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to Low level Intents",onfail="CHANGES WERE MADE TO LOW LEVEL INTENTS")
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ result = main.TRUE
+ flows2=[]
+ for i in range(27):
+ flows2.append(main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ result = result and main.Mininet2.flow_comp(flow1=flows[i], flow2=main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ if result == main.FALSE:
+ main.log.info("DIFFERENCES IN FLOW TABLES FOR SWITCH "+str(i))
+ break
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes in the flow tables",onfail="CHANGES IN THE FLOW TABLES!!")
+
+ main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+
+ def CASE7 (self,main):
+ main.case("Killing a link to Ensure that Link Discovery is Working Properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill Link between s3 and s28")
+ main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ time.sleep(5)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,str(int(links)-2))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link Down discovered properly",onfail="LINKS NOT DISCOVERED PROPERLY")
+ result1 = result
+ result = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+
+ main.step("Check for loss in pings when Link is brought down")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result2 = result
+ result = result1 and not result2
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link failure is discovered correctly and no traffic is lost!",onfail="Link Discovery failed or traffic was dropped!!!")
+
+ def CASE8 (self, main) :
+ import time
+ main.case("Killing a switch to ensure switch discovery is working properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill s28 ")
+ main.Mininet2.del_switch("s28")
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],str(int(active)-1),str(int(links)-4))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+
+ main.step("Add back s28")
+ main.Mininet2.add_switch("s28")
+ main.Mininet1.assign_sw_controller(sw="28",ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ main.Mininet1.assign_sw_controller(sw="28",count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,links)
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+ result1=result
+
+ main.step("Checking for Traffic Loss")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result = not result and result1
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovered Correctly and No Loss of traffic",onfail="Switch discovery failed or there was loss of traffic")
diff --git a/TestON/tests/HATestRCS/HATestRCS.topo b/TestON/tests/HATestRCS/HATestRCS.topo
new file mode 100644
index 0000000..9ccdb17
--- /dev/null
+++ b/TestON/tests/HATestRCS/HATestRCS.topo
@@ -0,0 +1,183 @@
+<TOPOLOGY>
+
+ <COMPONENT>
+ <ZK1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK1>
+
+ <ZK2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK2>
+
+ <ZK3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK3>
+
+ <ZK4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK4>
+
+ <ZK5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK5>
+
+ <RC1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC1>
+
+ <RC2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC2>
+
+ <RC3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC3>
+
+ <RC4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>9</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC4>
+
+ <RC5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC5>
+
+ <ONOS1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>11</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>12</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS3>
+
+ <ONOS4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS4>
+
+ <ONOS5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>15</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS5>
+
+ <Mininet1>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>MininetCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet1>
+
+ <Mininet2>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>RemoteMininetDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-onos4node.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet2>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestSuite/__init.py__ b/TestON/tests/HATestSuite/__init.py__
new file mode 100644
index 0000000..8d1c8b6
--- /dev/null
+++ b/TestON/tests/HATestSuite/__init.py__
@@ -0,0 +1 @@
+
diff --git a/TestON/tests/HATestTemplate/HATest1.params b/TestON/tests/HATestTemplate/HATest1.params
new file mode 100644
index 0000000..4d405f9
--- /dev/null
+++ b/TestON/tests/HATestTemplate/HATest1.params
@@ -0,0 +1,50 @@
+<PARAMS>
+ <testcases>2,3,4,5,6</testcases>
+ <CTRL>
+ <ip1>10.128.9.1</ip1>
+ <port1>6633</port1>
+ <restPort1>8080</restPort1>
+ <ip2>10.128.9.2</ip2>
+ <port2>6633</port2>
+ <restPort2>8080</restPort2>
+ <ip3>10.128.9.3</ip3>
+ <port3>6633</port3>
+ <restPort3>8080</restPort3>
+ <ip4>10.128.9.4</ip4>
+ <port4>6633</port4>
+ <restPort4>8080</restPort4>
+ <ip5>10.128.9.5</ip5>
+ <port5>6633</port5>
+ <restPort5>8080</restPort5>
+ <switchURL>/wm/onos/registry/switches/json</switchURL>
+ <intentHighURL>/wm/onos/intent/high</intentHighURL>
+ <intentLowURL>/wm/onos/intent/low</intentLowURL>
+ </CTRL>
+ <TopoRest>/wm/onos/topology</TopoRest>
+ <INTENTS>
+ <intentPort>8080</intentPort>
+ <intentURL>wm/onos/intent</intentURL>
+ </INTENTS>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+</PARAMS>
diff --git a/TestON/tests/HATestTemplate/HATest1.py b/TestON/tests/HATestTemplate/HATest1.py
new file mode 100644
index 0000000..92b4965
--- /dev/null
+++ b/TestON/tests/HATestTemplate/HATest1.py
@@ -0,0 +1,382 @@
+
+class HATest1:
+
+ global topology
+ global masterSwitchList
+ global highIntentList
+ global lowIntentList
+ global flows
+ flows = []
+
+ def __init__(self) :
+ self.default = ''
+
+ '''
+ CASE1 is to close any existing instances of ONOS, clean out the
+ RAMCloud database, and start up ONOS instances.
+ '''
+ def CASE1(self,main) :
+ main.case("Initial Startup")
+ main.step("Stop ONOS")
+ if not main.ONOS1.status():
+ main.ONOS1.stop_all()
+ if not main.ONOS1.status():
+ main.ONOS2.stop_all()
+ if not main.ONOS1.status():
+ main.ONOS3.stop_all()
+ if not main.ONOS1.status():
+ main.ONOS4.stop_all()
+ main.ONOS1.stop_rest()
+ main.ONOS2.stop_rest()
+ main.ONOS3.stop_rest()
+ main.ONOS4.stop_rest()
+ result = main.ONOS1.status() or main.ONOS2.status() \
+ or main.ONOS3.status() or main.ONOS4.status()
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="ONOS stopped successfully",onfail="ONOS WAS NOT KILLED PROPERLY")
+ main.step("Startup Zookeeper")
+ main.ZK1.start()
+ main.ZK2.start()
+ main.ZK3.start()
+ main.ZK4.start()
+ result = main.ZK1.isup() and main.ZK2.isup()\
+ and main.ZK3.isup() and main.ZK4.isup
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Zookeeper started successfully",onfail="ZOOKEEPER FAILED TO START")
+ main.step("Cleaning RC Database and Starting All")
+ main.RC1.deldb()
+ main.RC2.deldb()
+ main.RC3.deldb()
+ main.RC4.deldb()
+ main.ONOS1.start_all()
+ main.ONOS2.start_all()
+ main.ONOS3.start_all()
+ main.ONOS4.start_all()
+ main.ONOS1.start_rest()
+ main.step("Testing Startup")
+ result1 = main.ONOS1.rest_status()
+ vm1 = main.RC1.status_coor and main.RC1.status_serv and \
+ main.ONOS1.isup()
+ vm2 = main.RC2.status_coor and main.ONOS2.isup()
+ vm3 = main.RC3.status_coor and main.ONOS3.isup()
+ vm4 = main.RC4.status_coor and main.ONOS4.isup()
+ result = result1 and vm1 and vm2 and vm3 and vm4
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Everything started successfully",onfail="EVERYTHING FAILED TO START")
+
+ '''
+ CASE2
+ '''
+ def CASE2(self,main) :
+ import time
+ import json
+ import re
+ main.log.report("Assigning Controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign Master Controllers")
+ for i in range(1,28):
+ if i ==1:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ elif i>=2 and i<5:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip2'],port1=main.params['CTRL']['port2'])
+ elif i>=5 and i<8:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip3'],port1=main.params['CTRL']['port3'])
+ elif i>=8 and i<18:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip4'],port1=main.params['CTRL']['port4'])
+ elif i>=18 and i<28:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip5'],port1=main.params['CTRL']['port5'])
+ else:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+
+ result = main.TRUE
+ for i in range (1,28):
+ if i==1:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=2 and i<5:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip2'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=5 and i<8:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip3'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=8 and i<18:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip4'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=18 and i<28:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip5'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ else:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is" + str(response))
+ if re.search("tcp:" +main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ utilities.assert_equals(expect = main.TRUE,actual=result,onpass="MasterControllers assigned correctly")
+
+ def CASE3(self,main) :
+ import time
+ import json
+ import re
+ main.case("Adding Intents")
+ intentIP = main.params['CTRL']['ip1']
+ intentPort=main.params['INTENTS']['intentPort']
+ intentURL=main.params['INTENTS']['intentURL']
+ count = 1
+ for i in range(8,18):
+ srcMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ dstMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ srcDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ dstDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ dstDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ srcDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ dstMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ srcMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ count = 1
+ i = 8
+ result = main.TRUE
+ while i <18 :
+ main.log.info("\n\nh"+str(i)+" is Pinging h" + str(i+10))
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ if ping ==main.FALSE and count <9:
+ count+=1
+ i = 8
+ result = main.FALSE
+ main.log.info("Ping FAILED! Making attempt number "+str(count) + "in 2 seconds")
+ time.sleep(2)
+ elif ping==main.FALSE:
+ main.log.info("PINGS FAILED! MAX RETRIES REACHED!")
+ i=19
+ result = main.FALSE
+ elif ping==main.TRUE:
+ main.log.info("Ping passed!")
+ i+=1
+ result = main.TRUE
+ else:
+ main.log.info("ERROR!!")
+ result = main.ERROR
+ if result==main.FALSE:
+ main.log.info("INTENTS HAVE NOT BEEN INSTALLED CORRECTLY!! EXITING!!!")
+ main.cleanup()
+ main.exit()
+
+
+ def CASE4(self,main) :
+ import time
+ from subprocess import Popen, PIPE
+ main.case("Setting up and Gathering data for current state")
+ main.step("Get the current In-Memory Topology on each ONOS Instance")
+
+ '''
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip'+str(count)) in main.params['CTRL']:
+ temp = temp+(getattr(main,('ONOS'+str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count+=1
+ else:
+ break
+ topo_result = main.TRUE
+
+ for n in range(1,count):
+ temp_result = main.Mininet1.compare_topo(ctrls,main.ONOS1.get_json(main.params['CTRL']['ip'+str(n)]+":"+main.params['CTRL']['restPort'+str(n)]+main.params['TopoRest']))
+ '''
+
+ main.step("Get the Mastership of each switch")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ global masterSwitchList1
+ masterSwitchList1 = stdout
+
+ main.step("Get the High Level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ global highIntentList1
+ highIntentList1 = stdout
+
+ main.step("Get the Low level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ global lowIntentList1
+ lowIntentList1= stdout
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,28):
+ print main.Mininet2.get_flowTable("s"+str(i))
+ flows.append(main.Mininet2.get_flowTable("s"+str(i)))
+
+
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+
+ def CASE5(self,main) :
+ import re
+ main.case("MAIN COMPONENT FAILURE AND SCENARIO SPECIFIC TESTS")
+ main.step("Zookeeper Server Failure!")
+ result = main.TRUE
+ master1 = main.ZK1.status()
+ print master1
+ if re.search("leader",master1):
+ main.ZK1.stop()
+ main.log.info("ZK1 was Master and Killed! Also Killing ZK2")
+ main.ZK2.stop()
+ time.sleep(10)
+ if re.search("leader",main.ZK3.status()) or re.search("leader",main.ZK4.status()) or re.search("leader",main.ZK5.status()):
+ result = main.TRUE
+ main.log.info("New Leader Elected")
+ else:
+ result = main.FALSE
+ main.log.info("NO NEW ZK LEADER ELECTED!!!")
+ else:
+ master2 = main.ZK2.status()
+ if re.search("leader",master2):
+ main.ZK2.stop()
+ main.log.info("ZK2 was Master and Killed! Also Killing ZK3")
+ main.ZK3.stop()
+ time.sleep(10)
+ if re.search("leader",main.ZK1.status()) or re.search("leader",main.ZK4.status()) or re.search("leader",main.ZK5.status()):
+ result = main.TRUE
+ main.log.info("New Leader Elected")
+ else:
+ result = main.FALSE
+ main.log.info("NO NEW ZK LEADER ELECTED!!!")
+ else:
+ master3 = main.ZK3.status()
+ if re.search("leader",master3):
+ main.ZK3.stop()
+ main.log.info("ZK3 was Master and Killed! Also Killing ZK4")
+ main.ZK4.stop()
+ time.sleep(10)
+ if re.search("leader",main.ZK1.status()) or re.search("leader",main.ZK2.status()) or re.search("leader",main.ZK5.status()):
+ result = main.TRUE
+ main.log.info("New Leader Elected")
+ else:
+ result = main.FALSE
+ main.log.info("NO NEW ZK LEADER ELECTED!!!")
+ else:
+ master4 = main.ZK4.status()
+ if re.search("leader",master4):
+ main.ZK4.stop()
+ main.log.info("ZK4 was Master and Killed! Also Killing ZK5")
+ main.ZK5.stop()
+ time.sleep(10)
+ if re.search("leader",main.ZK1.status()) or re.search("leader",main.ZK2.status()) or re.search("leader",main.ZK3.status()):
+ result = main.TRUE
+ main.log.info("New Leader Elected")
+ else:
+ result = main.FALSE
+ main.log.info("NO NEW ZK LEADER ELECTED!!!")
+ else:
+ main.ZK5.stop()
+ main.log.info("ZK5 was Master and Killed! Also Killing ZK1")
+ main.ZK1.stop()
+ time.sleep(10)
+ if re.search("leader",main.ZK3.status()) or re.search("leader",main.ZK4.status()) or re.search("leader",main.ZK2.status()):
+ result = main.TRUE
+ main.log.info("New Leader Elected")
+ else:
+ result = main.FALSE
+ main.log.info("NO NEW ZK LEADER ELECTED!!!")
+
+
+ def CASE6(self,main) :
+ import os
+ main.case("Running ONOS Constant State Tests")
+ main.step("Get the current In-Memory Topology on each ONOS Instance and Compare it to the Topology before component failure")
+
+ main.step("Get the Mastership of each switch and compare to the Mastership before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ result = main.TRUE
+ for i in range(1,28):
+ if main.ZK1.findMaster(switchDPID="s"+str(i),switchList=masterSwitchList1)==main.ZK1.findMaster(switchDPID="s"+str(i),switchList=stdout):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Mastership of Switches was not changed",onfail="MASTERSHIP OF SWITCHES HAS CHANGED!!!")
+
+ main.step("Get the High Level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_intents(preIntents=highIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE HIGH LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to High level Intents",onfail="CHANGES WERE MADE TO HIGH LEVEL INTENTS")
+
+ main.step("Get the Low level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_low(preIntents=lowIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE LOW LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to Low level Intents",onfail="CHANGES WERE MADE TO LOW LEVEL INTENTS")
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ result = main.TRUE
+ flows2=[]
+ for i in range(27):
+ flows2.append(main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ result = result and main.Mininet2.flow_comp(flow1=flows[i], flow2=main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ print flows[i]
+ print flows2[i]
+ if result == main.FALSE:
+ main.log.info("DIFFERENCES IN FLOW TABLES FOR SWITCH "+str(i))
+ break
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes in the flow tables",onfail="CHANGES IN THE FLOW TABLES!!")
+
+ main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+
+
+
+
+
diff --git a/TestON/tests/HATestTemplate/HATest1.topo b/TestON/tests/HATestTemplate/HATest1.topo
new file mode 100644
index 0000000..9ccdb17
--- /dev/null
+++ b/TestON/tests/HATestTemplate/HATest1.topo
@@ -0,0 +1,183 @@
+<TOPOLOGY>
+
+ <COMPONENT>
+ <ZK1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK1>
+
+ <ZK2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK2>
+
+ <ZK3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK3>
+
+ <ZK4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK4>
+
+ <ZK5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK5>
+
+ <RC1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC1>
+
+ <RC2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC2>
+
+ <RC3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC3>
+
+ <RC4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>9</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC4>
+
+ <RC5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC5>
+
+ <ONOS1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>11</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>12</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS3>
+
+ <ONOS4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS4>
+
+ <ONOS5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>15</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS5>
+
+ <Mininet1>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>MininetCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet1>
+
+ <Mininet2>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>RemoteMininetDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-onos4node.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet2>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestZK/HATestZK.params b/TestON/tests/HATestZK/HATestZK.params
index 677605c..09a6a3d 100644
--- a/TestON/tests/HATestZK/HATestZK.params
+++ b/TestON/tests/HATestZK/HATestZK.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,3,4,5,6</testcases>
+ <testcases>2,3,4,5,6,7,8</testcases>
<CTRL>
<ip1>10.128.9.1</ip1>
<port1>6633</port1>
diff --git a/TestON/tests/HATestZK/HATestZK.py b/TestON/tests/HATestZK/HATestZK.py
index 89969d3..afcd020 100644
--- a/TestON/tests/HATestZK/HATestZK.py
+++ b/TestON/tests/HATestZK/HATestZK.py
@@ -18,39 +18,40 @@
def CASE1(self,main) :
main.case("Initial Startup")
main.step("Stop ONOS")
- if not main.ONOS1.status():
- main.ONOS1.stop_all()
- if not main.ONOS1.status():
- main.ONOS2.stop_all()
- if not main.ONOS1.status():
- main.ONOS3.stop_all()
- if not main.ONOS1.status():
- main.ONOS4.stop_all()
+ main.ONOS1.stop_all()
+ main.ONOS2.stop_all()
+ main.ONOS3.stop_all()
+ main.ONOS4.stop_all()
+ main.ONOS5.stop_all()
main.ONOS1.stop_rest()
main.ONOS2.stop_rest()
main.ONOS3.stop_rest()
main.ONOS4.stop_rest()
+ main.ONOS5.stop_rest()
result = main.ONOS1.status() or main.ONOS2.status() \
- or main.ONOS3.status() or main.ONOS4.status()
+ or main.ONOS3.status() or main.ONOS4.status() or main.ONOS5.status()
utilities.assert_equals(expect=main.FALSE,actual=result,onpass="ONOS stopped successfully",onfail="ONOS WAS NOT KILLED PROPERLY")
main.step("Startup Zookeeper")
main.ZK1.start()
main.ZK2.start()
main.ZK3.start()
main.ZK4.start()
+ main.ZK5.start()
result = main.ZK1.isup() and main.ZK2.isup()\
- and main.ZK3.isup() and main.ZK4.isup
+ and main.ZK3.isup() and main.ZK4.isup() and main.ZK5.isup()
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Zookeeper started successfully",onfail="ZOOKEEPER FAILED TO START")
main.step("Cleaning RC Database and Starting All")
- main.RC1.deldb()
- main.RC2.deldb()
- main.RC3.deldb()
- main.RC4.deldb()
+ main.RC1.del_db()
+ main.RC2.del_db()
+ main.RC3.del_db()
+ main.RC4.del_db()
+ main.RC5.del_db()
main.ONOS1.start_all()
main.ONOS2.start_all()
main.ONOS3.start_all()
main.ONOS4.start_all()
- main.ONOS1.start_rest()
+ main.ONOS5.start_all()
+ # main.ONOS1.start_rest()
main.step("Testing Startup")
result1 = main.ONOS1.rest_status()
vm1 = main.RC1.status_coor and main.RC1.status_serv and \
@@ -58,8 +59,12 @@
vm2 = main.RC2.status_coor and main.ONOS2.isup()
vm3 = main.RC3.status_coor and main.ONOS3.isup()
vm4 = main.RC4.status_coor and main.ONOS4.isup()
- result = result1 and vm1 and vm2 and vm3 and vm4
+ vm5 = main.RC5.status_coor and main.ONOS5.isup()
+ result = result1 and vm1 and vm2 and vm3 and vm4 and vm5
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Everything started successfully",onfail="EVERYTHING FAILED TO START")
+ if result==main.FALSE:
+ main.cleanup()
+ main.exit()
'''
CASE2
@@ -71,7 +76,7 @@
main.log.report("Assigning Controllers")
main.case("Assigning Controllers")
main.step("Assign Master Controllers")
- for i in range(1,28):
+ for i in range(1,29):
if i ==1:
main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
elif i>=2 and i<5:
@@ -86,7 +91,7 @@
main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
result = main.TRUE
- for i in range (1,28):
+ for i in range (1,29):
if i==1:
response = main.Mininet1.get_sw_controller("s"+str(i))
print("Response is " + str(response))
@@ -129,7 +134,10 @@
result = result and main.TRUE
else:
result = main.FALSE
+
utilities.assert_equals(expect = main.TRUE,actual=result,onpass="MasterControllers assigned correctly")
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
def CASE3(self,main) :
import time
@@ -226,8 +234,7 @@
main.step("Get the OF Table entries")
global flows
flows=[]
- for i in range(1,28):
- print main.Mininet2.get_flowTable("s"+str(i))
+ for i in range(1,29):
flows.append(main.Mininet2.get_flowTable("s"+str(i)))
@@ -312,6 +319,8 @@
else:
result = main.FALSE
main.log.info("NO NEW ZK LEADER ELECTED!!!")
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="New Leader was Elected!",onfail="NO NEW LEADER WAS ELECTED!!!!")
+
def CASE6(self,main) :
@@ -322,12 +331,17 @@
main.step("Get the Mastership of each switch and compare to the Mastership before component failure")
(stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
result = main.TRUE
- for i in range(1,28):
- if main.ZK1.findMaster(switchDPID="s"+str(i),switchList=masterSwitchList1)==main.ZK1.findMaster(switchDPID="s"+str(i),switchList=stdout):
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+ switchDPID = switchDPID[:2]+":"+switchDPID[2:4]+":"+switchDPID[4:6]+":"+switchDPID[6:8]+":"+switchDPID[8:10]+":"+switchDPID[10:12]+":"+switchDPID[12:14]+":"+switchDPID[14:]
+ master1 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)
+ master2 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout)
+ if main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)==main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout):
result = result and main.TRUE
else:
result = main.FALSE
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Mastership of Switches was not changed",onfail="MASTERSHIP OF SWITCHES HAS CHANGED!!!")
+ result1 = result
main.step("Get the High Level Intents and compare to before component failure")
(stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
@@ -338,6 +352,7 @@
main.log.info("THERE WERE CHANGES TO THE HIGH LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
result = main.FALSE
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to High level Intents",onfail="CHANGES WERE MADE TO HIGH LEVEL INTENTS")
+ result2=result
main.step("Get the Low level Intents and compare to before component failure")
(stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
@@ -348,6 +363,7 @@
main.log.info("THERE WERE CHANGES TO THE LOW LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
result = main.FALSE
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to Low level Intents",onfail="CHANGES WERE MADE TO LOW LEVEL INTENTS")
+ result3=result
main.step("Get the OF Table entries and compare to before component failure")
@@ -355,13 +371,12 @@
flows2=[]
for i in range(27):
flows2.append(main.Mininet2.get_flowTable(sw="s"+str(i+1)))
- result = result and main.Mininet2.flow_comp(flow1=flows[i], flow2=main.Mininet2.get_flowTable(sw="s"+str(i+1)))
- print flows[i]
- print flows2[i]
+ result = result and main.Mininet2.flow_comp(flow1=flows[i], flow2=main.Mininet2.get_flowTable(sw="s"+str(i+1)))
if result == main.FALSE:
main.log.info("DIFFERENCES IN FLOW TABLES FOR SWITCH "+str(i))
break
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes in the flow tables",onfail="CHANGES IN THE FLOW TABLES!!")
+ result4 = result
main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
main.Mininet2.pingKill()
@@ -375,8 +390,101 @@
else:
main.log.info("No Loss in the pings!")
utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result5=not result
+ result = result1 and result2 and result3 and result4 and result5
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Constant State Tests Passed!",onfail="CONSTANT STATE TESTS FAILED!!")
+ def CASE7 (self,main):
+ main.case("Killing a link to Ensure that Link Discovery is Working Properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill Link between s3 and s28")
+ main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,str(int(links)-2))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link Down discovered properly",onfail="LINKS NOT DISCOVERED PROPERLY")
+ result1 = result
+ result = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+ main.step("Check for loss in pings when Link is brought down")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result2 = result
+ result = result1 and not result2
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="Link failure is discovered correctly and no traffic is lost!",onfail="Link Discovery failed or traffic was dropped!!!")
+
+
+ def CASE8 (self, main) :
+ import time
+ main.case("Killing a switch to ensure switch discovery is working properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill s28 ")
+ main.Mininet2.del_switch("s28")
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],str(int(active)-1),str(int(links)-4))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+
+ main.step("Add back s28")
+ main.Mininet2.add_switch("s28")
+ main.Mininet1.assign_sw_controller(sw="28",ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ main.Mininet1.assign_sw_controller(sw="28",count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,links)
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+ result1=result
+
+ main.step("Checking for Traffic Loss")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result = not result and result1
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovered Correctly and No Loss of traffic",onfail="Switch discovery failed or there was loss of traffic")
diff --git a/TestON/tests/HATestZK2/HATestZK2.params b/TestON/tests/HATestZK2/HATestZK2.params
new file mode 100644
index 0000000..6952d1b
--- /dev/null
+++ b/TestON/tests/HATestZK2/HATestZK2.params
@@ -0,0 +1,50 @@
+<PARAMS>
+ <testcases>1,2,3,4,5,6,7,8</testcases>
+ <CTRL>
+ <ip1>10.128.9.1</ip1>
+ <port1>6633</port1>
+ <restPort1>8080</restPort1>
+ <ip2>10.128.9.2</ip2>
+ <port2>6633</port2>
+ <restPort2>8080</restPort2>
+ <ip3>10.128.9.3</ip3>
+ <port3>6633</port3>
+ <restPort3>8080</restPort3>
+ <ip4>10.128.9.4</ip4>
+ <port4>6633</port4>
+ <restPort4>8080</restPort4>
+ <ip5>10.128.9.5</ip5>
+ <port5>6633</port5>
+ <restPort5>8080</restPort5>
+ <switchURL>/wm/onos/registry/switches/json</switchURL>
+ <intentHighURL>/wm/onos/intent/high</intentHighURL>
+ <intentLowURL>/wm/onos/intent/low</intentLowURL>
+ </CTRL>
+ <TopoRest>/wm/onos/topology</TopoRest>
+ <INTENTS>
+ <intentPort>8080</intentPort>
+ <intentURL>wm/onos/intent</intentURL>
+ </INTENTS>
+ <PING>
+ <source1>h8</source1>
+ <source2>h9</source2>
+ <source3>h10</source3>
+ <source4>h11</source4>
+ <source5>h12</source5>
+ <source6>h13</source6>
+ <source7>h14</source7>
+ <source8>h15</source8>
+ <source9>h16</source9>
+ <source10>h17</source10>
+ <target1>10.0.0.18</target1>
+ <target2>10.0.0.19</target2>
+ <target3>10.0.0.20</target3>
+ <target4>10.0.0.21</target4>
+ <target5>10.0.0.22</target5>
+ <target6>10.0.0.23</target6>
+ <target7>10.0.0.24</target7>
+ <target8>10.0.0.25</target8>
+ <target9>10.0.0.26</target9>
+ <target10>10.0.0.27</target10>
+ </PING>
+</PARAMS>
diff --git a/TestON/tests/HATestZK2/HATestZK2.py b/TestON/tests/HATestZK2/HATestZK2.py
new file mode 100644
index 0000000..31db5f9
--- /dev/null
+++ b/TestON/tests/HATestZK2/HATestZK2.py
@@ -0,0 +1,459 @@
+
+class HATestZK2:
+
+ global topology
+ global masterSwitchList
+ global highIntentList
+ global lowIntentList
+ global flows
+ flows = []
+
+ def __init__(self) :
+ self.default = ''
+
+ '''
+ CASE1 is to close any existing instances of ONOS, clean out the
+ RAMCloud database, and start up ONOS instances.
+ '''
+ def CASE1(self,main) :
+ main.case("Initial Startup")
+ main.step("Stop ONOS")
+ main.ONOS1.stop_all()
+ main.ONOS2.stop_all()
+ main.ONOS3.stop_all()
+ main.ONOS4.stop_all()
+ main.ONOS5.stop_all()
+ main.ONOS1.stop_rest()
+ main.ONOS2.stop_rest()
+ main.ONOS3.stop_rest()
+ main.ONOS4.stop_rest()
+ main.ONOS5.stop_rest()
+ result = main.ONOS1.status() or main.ONOS2.status() \
+ or main.ONOS3.status() or main.ONOS4.status() or main.ONOS5.status()
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="ONOS stopped successfully",onfail="ONOS WAS NOT KILLED PROPERLY")
+ main.step("Startup Zookeeper")
+ main.ZK1.start()
+ main.ZK2.start()
+ main.ZK3.start()
+ main.ZK4.start()
+ main.ZK5.start()
+ result = main.ZK1.isup() and main.ZK2.isup()\
+ and main.ZK3.isup() and main.ZK4.isup() and main.ZK5.isup()
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Zookeeper started successfully",onfail="ZOOKEEPER FAILED TO START")
+ main.step("Cleaning RC Database and Starting All")
+ main.RC1.del_db()
+ main.RC2.del_db()
+ main.RC3.del_db()
+ main.RC4.del_db()
+ main.RC5.del_db()
+ main.ONOS1.start_all()
+ main.ONOS2.start_all()
+ main.ONOS3.start_all()
+ main.ONOS4.start_all()
+ main.ONOS5.start_all()
+ # main.ONOS1.start_rest()
+ main.step("Testing Startup")
+ result1 = main.ONOS1.rest_status()
+ vm1 = main.RC1.status_coor and main.RC1.status_serv and \
+ main.ONOS1.isup()
+ vm2 = main.RC2.status_coor and main.ONOS2.isup()
+ vm3 = main.RC3.status_coor and main.ONOS3.isup()
+ vm4 = main.RC4.status_coor and main.ONOS4.isup()
+ vm5 = main.RC5.status_coor and main.ONOS5.isup()
+ result = result1 and vm1 and vm2 and vm3 and vm4 and vm5
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Everything started successfully",onfail="EVERYTHING FAILED TO START")
+ if result==main.FALSE:
+ main.cleanup()
+ main.exit()
+
+ '''
+ CASE2
+ '''
+ def CASE2(self,main) :
+ import time
+ import json
+ import re
+ main.log.report("Assigning Controllers")
+ main.case("Assigning Controllers")
+ main.step("Assign Master Controllers")
+ for i in range(1,29):
+ if i ==1:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ elif i>=2 and i<5:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip2'],port1=main.params['CTRL']['port2'])
+ elif i>=5 and i<8:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip3'],port1=main.params['CTRL']['port3'])
+ elif i>=8 and i<18:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip4'],port1=main.params['CTRL']['port4'])
+ elif i>=18 and i<28:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip5'],port1=main.params['CTRL']['port5'])
+ else:
+ main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+
+ result = main.TRUE
+ for i in range (1,29):
+ if i==1:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=2 and i<5:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip2'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=5 and i<8:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip3'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=8 and i<18:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip4'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ elif i>=18 and i<28:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is " + str(response))
+ if re.search("tcp:"+main.params['CTRL']['ip5'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ else:
+ response = main.Mininet1.get_sw_controller("s"+str(i))
+ print("Response is" + str(response))
+ if re.search("tcp:" +main.params['CTRL']['ip1'],response):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+
+ utilities.assert_equals(expect = main.TRUE,actual=result,onpass="MasterControllers assigned correctly")
+ for i in range (1,29):
+ main.Mininet1.assign_sw_controller(sw=str(i),count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+
+ def CASE3(self,main) :
+ import time
+ import json
+ import re
+ main.case("Adding Intents")
+ intentIP = main.params['CTRL']['ip1']
+ intentPort=main.params['INTENTS']['intentPort']
+ intentURL=main.params['INTENTS']['intentURL']
+ count = 1
+ for i in range(8,18):
+ srcMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ dstMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ srcDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ dstDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ dstDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
+ srcDPID= '00:00:00:00:00:00:60:' +str(i+10)
+ dstMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
+ srcMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
+ main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
+ count+=1
+ count = 1
+ i = 8
+ result = main.TRUE
+ while i <18 :
+ main.log.info("\n\nh"+str(i)+" is Pinging h" + str(i+10))
+ ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
+ if ping ==main.FALSE and count <9:
+ count+=1
+ i = 8
+ result = main.FALSE
+ main.log.info("Ping FAILED! Making attempt number "+str(count) + "in 2 seconds")
+ time.sleep(2)
+ elif ping==main.FALSE:
+ main.log.info("PINGS FAILED! MAX RETRIES REACHED!")
+ i=19
+ result = main.FALSE
+ elif ping==main.TRUE:
+ main.log.info("Ping passed!")
+ i+=1
+ result = main.TRUE
+ else:
+ main.log.info("ERROR!!")
+ result = main.ERROR
+ if result==main.FALSE:
+ main.log.info("INTENTS HAVE NOT BEEN INSTALLED CORRECTLY!! EXITING!!!")
+ main.cleanup()
+ main.exit()
+
+
+ def CASE4(self,main) :
+ import time
+ from subprocess import Popen, PIPE
+ main.case("Setting up and Gathering data for current state")
+ main.step("Get the current In-Memory Topology on each ONOS Instance")
+
+ '''
+ ctrls = []
+ count = 1
+ while True:
+ temp = ()
+ if ('ip'+str(count)) in main.params['CTRL']:
+ temp = temp+(getattr(main,('ONOS'+str(count))),)
+ temp = temp + ("ONOS"+str(count),)
+ temp = temp + (main.params['CTRL']['ip'+str(count)],)
+ temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
+ ctrls.append(temp)
+ count+=1
+ else:
+ break
+ topo_result = main.TRUE
+
+ for n in range(1,count):
+ temp_result = main.Mininet1.compare_topo(ctrls,main.ONOS1.get_json(main.params['CTRL']['ip'+str(n)]+":"+main.params['CTRL']['restPort'+str(n)]+main.params['TopoRest']))
+ '''
+
+ main.step("Get the Mastership of each switch")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ global masterSwitchList1
+ masterSwitchList1 = stdout
+
+ main.step("Get the High Level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ global highIntentList1
+ highIntentList1 = stdout
+
+ main.step("Get the Low level Intents")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ global lowIntentList1
+ lowIntentList1= stdout
+
+ main.step("Get the OF Table entries")
+ global flows
+ flows=[]
+ for i in range(1,29):
+ flows.append(main.Mininet2.get_flowTable("s"+str(i)))
+
+
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+
+ def CASE5(self,main) :
+ import re
+ main.case("MAIN COMPONENT FAILURE AND SCENARIO SPECIFIC TESTS")
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+
+ main.step("Zookeeper Server Failure!")
+ main.ZK1.stop()
+ main.ZK2.stop()
+ main.ZK3.stop()
+ main.ZK4.stop()
+ main.ZK5.stop()
+ main.Mininet2.del_switch("s1")
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],str(int(active)-1),str(int(links)-2))
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="Registry is no longer active",onfail="Registry is still being updated")
+
+ main.step("Restart Zookeeper")
+ main.ZK1.start()
+ main.ZK2.start()
+ main.ZK3.start()
+ main.ZK4.start()
+ main.ZK5.start()
+ time.sleep(10) # Time for Zookeeper to reboot
+ master1=main.ZK1.status()
+ master2=main.ZK2.status()
+ master3=main.ZK3.status()
+ master4=main.ZK4.status()
+ master5=main.ZK5.status()
+ if re.search("leader",master1) or re.search("leader",master2) or re.search("leader",master3) or re.search("leader",master4) or re.search("leader",master5):
+ main.log.info("New ZK Leader Elected")
+ else:
+ main.log.info("NO NEW ZK LEADER ELECTED!!!")
+ main.step("Add back s1")
+ main.Mininet2.add_switch("s1")
+ main.Mininet1.assign_sw_controller(sw="1",ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ main.Mininet1.assign_sw_controller(sw="1",count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+ time.sleep(31)
+
+
+ def CASE6(self,main) :
+ import os
+ main.case("Running ONOS Constant State Tests")
+ main.step("Get the current In-Memory Topology on each ONOS Instance and Compare it to the Topology before component failure")
+
+ main.step("Get the Mastership of each switch and compare to the Mastership before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
+ result = main.TRUE
+ for i in range(1,29):
+ switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
+ switchDPID = switchDPID[:2]+":"+switchDPID[2:4]+":"+switchDPID[4:6]+":"+switchDPID[6:8]+":"+switchDPID[8:10]+":"+switchDPID[10:12]+":"+switchDPID[12:14]+":"+switchDPID[14:]
+ master1 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)
+ master2 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout)
+ if main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)==main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout):
+ result = result and main.TRUE
+ else:
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Mastership of Switches was not changed",onfail="MASTERSHIP OF SWITCHES HAS CHANGED!!!")
+ result1 = result
+
+ main.step("Get the High Level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_intents(preIntents=highIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE HIGH LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to High level Intents",onfail="CHANGES WERE MADE TO HIGH LEVEL INTENTS")
+ result2=result
+
+ main.step("Get the Low level Intents and compare to before component failure")
+ (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
+ changesInIntents=main.ONOS1.comp_low(preIntents=lowIntentList1,postIntents=stdout)
+ if not changesInIntents:
+ result = main.TRUE
+ else:
+ main.log.info("THERE WERE CHANGES TO THE LOW LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
+ result = main.FALSE
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to Low level Intents",onfail="CHANGES WERE MADE TO LOW LEVEL INTENTS")
+ result3=result
+
+
+ main.step("Get the OF Table entries and compare to before component failure")
+ result = main.TRUE
+ flows2=[]
+ for i in range(27):
+ flows2.append(main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ result = result and main.Mininet2.flow_comp(flow1=flows[i], flow2=main.Mininet2.get_flowTable(sw="s"+str(i+1)))
+ if result == main.FALSE:
+ main.log.info("DIFFERENCES IN FLOW TABLES FOR SWITCH "+str(i))
+ break
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes in the flow tables",onfail="CHANGES IN THE FLOW TABLES!!")
+ result4 = result
+
+ main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result5=not result
+ result = result1 and result2 and result3 and result4 and result5
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Constant State Tests Passed!",onfail="CONSTANT STATE TESTS FAILED!!")
+
+ def CASE7 (self,main):
+ main.case("Killing a link to Ensure that Link Discovery is Working Properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill Link between s3 and s28")
+ main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,str(int(links)-2))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link Down discovered properly",onfail="LINKS NOT DISCOVERED PROPERLY")
+ result1 = result
+ result = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
+
+ main.step("Check for loss in pings when Link is brought down")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result2 = result
+ result = result1 and not result2
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="Link failure is discovered correctly and no traffic is lost!",onfail="Link Discovery failed or traffic was dropped!!!")
+
+
+
+ def CASE8 (self, main) :
+ import time
+ main.case("Killing a switch to ensure switch discovery is working properly")
+ main.step("Start continuous pings")
+ main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
+ main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
+
+ main.step("Determine the current number of switches and links")
+ (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
+ links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
+ main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
+
+ main.step("Kill s28 ")
+ main.Mininet2.del_switch("s28")
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],str(int(active)-1),str(int(links)-4))
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+
+ main.step("Add back s28")
+ main.Mininet2.add_switch("s28")
+ main.Mininet1.assign_sw_controller(sw="28",ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+ main.Mininet1.assign_sw_controller(sw="28",count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
+ time.sleep(31)
+ result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,links)
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
+ result1=result
+
+ main.step("Checking for Traffic Loss")
+ main.Mininet2.pingKill()
+ result = main.FALSE
+ for i in range(8,18):
+ result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
+ if result==main.TRUE:
+ main.log.info("LOSS IN THE PINGS!")
+ elif result == main.ERROR:
+ main.log.info("There are multiple mininet process running!!")
+ else:
+ main.log.info("No Loss in the pings!")
+ utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
+ result = not result and result1
+ utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovered Correctly and No Loss of traffic",onfail="Switch discovery failed or there was loss of traffic")
+
diff --git a/TestON/tests/HATestZK2/HATestZK2.topo b/TestON/tests/HATestZK2/HATestZK2.topo
new file mode 100644
index 0000000..9ccdb17
--- /dev/null
+++ b/TestON/tests/HATestZK2/HATestZK2.topo
@@ -0,0 +1,183 @@
+<TOPOLOGY>
+
+ <COMPONENT>
+ <ZK1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK1>
+
+ <ZK2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK2>
+
+ <ZK3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK3>
+
+ <ZK4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK4>
+
+ <ZK5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>ZookeeperCliDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ZK5>
+
+ <RC1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC1>
+
+ <RC2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC2>
+
+ <RC3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC3>
+
+ <RC4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>9</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC4>
+
+ <RC5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>RamCloudCliDriver</type>
+ <connect_order>10</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </RC5>
+
+ <ONOS1>
+ <host>10.128.9.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>11</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.9.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>12</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.9.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS3>
+
+ <ONOS4>
+ <host>10.128.9.4</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS4>
+
+ <ONOS5>
+ <host>10.128.9.5</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>15</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOS5>
+
+ <Mininet1>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>MininetCliDriver</type>
+ <connect_order>13</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet1>
+
+ <Mininet2>
+ <host>10.128.9.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>RemoteMininetDriver</type>
+ <connect_order>14</connect_order>
+ <COMPONENTS>
+ # Specify the Option for mininet
+ <arg1> --custom ~/mininet/custom/topo-onos4node.py </arg1>
+ <arg2> --topo mytopo --arp</arg2>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </Mininet2>
+
+ </COMPONENT>
+</TOPOLOGY>