Gitiles
Code Review Sign In
gerrit.onosproject.org / OnosSystemTest / fe0c27dee77018a937013a88ea1a02467faf6913 / . / TestON / tests / HATestONOS / HATestONOS.py
blob: ff9c7b307cba61f460f8e0c17172ef4cae8428cf [file] [log] [blame]
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]1
2class HATestONOS:
3
4 global topology
5 global masterSwitchList
6 global highIntentList
7 global lowIntentList
8 global flows
9 flows = []
10
11 def __init__(self) :
12 self.default = ''
13
14 '''
15 CASE1 is to close any existing instances of ONOS, clean out the
16 RAMCloud database, and start up ONOS instances.
17 '''
18 def CASE1(self,main) :
19 main.case("Initial Startup")
20 main.step("Stop ONOS")
21 main.ONOS1.stop_all()
22 main.ONOS2.stop_all()
23 main.ONOS3.stop_all()
24 main.ONOS4.stop_all()
25 main.ONOS5.stop_all()
26 main.ONOS1.stop_rest()
27 main.ONOS2.stop_rest()
28 main.ONOS3.stop_rest()
29 main.ONOS4.stop_rest()
30 main.ONOS5.stop_rest()
31 result = main.ONOS1.status() or main.ONOS2.status() \
32 or main.ONOS3.status() or main.ONOS4.status() or main.ONOS5.status()
33 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="ONOS stopped successfully",onfail="ONOS WAS NOT KILLED PROPERLY")
34 main.step("Startup Zookeeper")
35 main.ZK1.start()
36 main.ZK2.start()
37 main.ZK3.start()
38 main.ZK4.start()
39 main.ZK5.start()
40 result = main.ZK1.isup() and main.ZK2.isup()\
41 and main.ZK3.isup() and main.ZK4.isup() and main.ZK5.isup()
42 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Zookeeper started successfully",onfail="ZOOKEEPER FAILED TO START")
43 main.step("Cleaning RC Database and Starting All")
44 main.RC1.del_db()
45 main.RC2.del_db()
46 main.RC3.del_db()
47 main.RC4.del_db()
48 main.RC5.del_db()
49 main.ONOS1.start_all()
50 main.ONOS2.start_all()
51 main.ONOS3.start_all()
52 main.ONOS4.start_all()
53 main.ONOS5.start_all()
54 # main.ONOS1.start_rest()
55 main.step("Testing Startup")
56 result1 = main.ONOS1.rest_status()
57 vm1 = main.RC1.status_coor and main.RC1.status_serv and \
58 main.ONOS1.isup()
59 vm2 = main.RC2.status_coor and main.ONOS2.isup()
60 vm3 = main.RC3.status_coor and main.ONOS3.isup()
61 vm4 = main.RC4.status_coor and main.ONOS4.isup()
62 vm5 = main.RC5.status_coor and main.ONOS5.isup()
63 result = result1 and vm1 and vm2 and vm3 and vm4 and vm5
64 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Everything started successfully",onfail="EVERYTHING FAILED TO START")
65
66 '''
67 CASE2
68 '''
69 def CASE2(self,main) :
70 import time
71 import json
72 import re
73 main.log.report("Assigning Controllers")
74 main.case("Assigning Controllers")
75 main.step("Assign Master Controllers")
76 for i in range(1,29):
77 if i ==1:
78 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
79 elif i>=2 and i<5:
80 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip2'],port1=main.params['CTRL']['port2'])
81 elif i>=5 and i<8:
82 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip3'],port1=main.params['CTRL']['port3'])
83 elif i>=8 and i<18:
84 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip4'],port1=main.params['CTRL']['port4'])
85 elif i>=18 and i<28:
86 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip5'],port1=main.params['CTRL']['port5'])
87 else:
88 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
89
90 result = main.TRUE
91 for i in range (1,29):
92 if i==1:
93 response = main.Mininet1.get_sw_controller("s"+str(i))
94 print("Response is " + str(response))
95 if re.search("tcp:"+main.params['CTRL']['ip1'],response):
96 result = result and main.TRUE
97 else:
98 result = main.FALSE
99 elif i>=2 and i<5:
100 response = main.Mininet1.get_sw_controller("s"+str(i))
101 print("Response is " + str(response))
102 if re.search("tcp:"+main.params['CTRL']['ip2'],response):
103 result = result and main.TRUE
104 else:
105 result = main.FALSE
106 elif i>=5 and i<8:
107 response = main.Mininet1.get_sw_controller("s"+str(i))
108 print("Response is " + str(response))
109 if re.search("tcp:"+main.params['CTRL']['ip3'],response):
110 result = result and main.TRUE
111 else:
112 result = main.FALSE
113 elif i>=8 and i<18:
114 response = main.Mininet1.get_sw_controller("s"+str(i))
115 print("Response is " + str(response))
116 if re.search("tcp:"+main.params['CTRL']['ip4'],response):
117 result = result and main.TRUE
118 else:
119 result = main.FALSE
120 elif i>=18 and i<28:
121 response = main.Mininet1.get_sw_controller("s"+str(i))
122 print("Response is " + str(response))
123 if re.search("tcp:"+main.params['CTRL']['ip5'],response):
124 result = result and main.TRUE
125 else:
126 result = main.FALSE
127 else:
128 response = main.Mininet1.get_sw_controller("s"+str(i))
129 print("Response is" + str(response))
130 if re.search("tcp:" +main.params['CTRL']['ip1'],response):
131 result = result and main.TRUE
132 else:
133 result = main.FALSE
134
135 utilities.assert_equals(expect = main.TRUE,actual=result,onpass="MasterControllers assigned correctly")
136 for i in range (1,29):
137 main.Mininet1.assign_sw_controller(sw=str(i),count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
138
139 def CASE3(self,main) :
140 import time
141 import json
142 import re
143 main.case("Adding Intents")
144 intentIP = main.params['CTRL']['ip1']
145 intentPort=main.params['INTENTS']['intentPort']
146 intentURL=main.params['INTENTS']['intentURL']
147 count = 1
148 for i in range(8,18):
149 srcMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
150 dstMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
151 srcDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
152 dstDPID= '00:00:00:00:00:00:60:' +str(i+10)
153 main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
154 count+=1
155 dstDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
156 srcDPID= '00:00:00:00:00:00:60:' +str(i+10)
157 dstMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
158 srcMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
159 main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
160 count+=1
161 count = 1
162 i = 8
163 result = main.TRUE
164 while i <18 :
165 main.log.info("\n\nh"+str(i)+" is Pinging h" + str(i+10))
166 ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
167 if ping ==main.FALSE and count <9:
168 count+=1
169 i = 8
170 result = main.FALSE
171 main.log.info("Ping FAILED! Making attempt number "+str(count) + "in 2 seconds")
172 time.sleep(2)
173 elif ping==main.FALSE:
174 main.log.info("PINGS FAILED! MAX RETRIES REACHED!")
175 i=19
176 result = main.FALSE
177 elif ping==main.TRUE:
178 main.log.info("Ping passed!")
179 i+=1
180 result = main.TRUE
181 else:
182 main.log.info("ERROR!!")
183 result = main.ERROR
184 if result==main.FALSE:
185 main.log.info("INTENTS HAVE NOT BEEN INSTALLED CORRECTLY!! EXITING!!!")
186 main.cleanup()
187 main.exit()
188
189
190 def CASE4(self,main) :
191 import time
192 from subprocess import Popen, PIPE
193 main.case("Setting up and Gathering data for current state")
194 main.step("Get the current In-Memory Topology on each ONOS Instance")
195
196 '''
197 ctrls = []
198 count = 1
199 while True:
200 temp = ()
201 if ('ip'+str(count)) in main.params['CTRL']:
202 temp = temp+(getattr(main,('ONOS'+str(count))),)
203 temp = temp + ("ONOS"+str(count),)
204 temp = temp + (main.params['CTRL']['ip'+str(count)],)
205 temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
206 ctrls.append(temp)
207 count+=1
208 else:
209 break
210 topo_result = main.TRUE
211
212 for n in range(1,count):
213 temp_result = main.Mininet1.compare_topo(ctrls,main.ONOS1.get_json(main.params['CTRL']['ip'+str(n)]+":"+main.params['CTRL']['restPort'+str(n)]+main.params['TopoRest']))
214 '''
215 '''
216 main.step("Get the Mastership of each switch")
217 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
218 global masterSwitchList1
219 masterSwitchList1 = stdout
220 '''
221
222 main.step("Get the High Level Intents")
223 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
224 global highIntentList1
225 highIntentList1 = stdout
226
227 main.step("Get the Low level Intents")
228 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
229 global lowIntentList1
230 lowIntentList1= stdout
231
232 main.step("Get the OF Table entries")
233 global flows
234 flows=[]
235 for i in range(1,29):
236 flows.append(main.Mininet2.get_flowTable("s"+str(i)))
237
238
239 main.step("Start continuous pings")
240 main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
241 main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
242 main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
243 main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
244 main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
245 main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
246 main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
247 main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
248 main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
249 main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
250
251
252 def CASE5(self,main) :
253 import re
254 from random import randint
255 main.case("MAIN COMPONENT FAILURE AND SCENARIO SPECIFIC TESTS")
256 main.step("ONOS CORE Failure!")
257 kill = randint(1,4)
258 if kill==1:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]259 main.ONOS2.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]260 elif kill==2:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]261 main.ONOS2.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]262 elif kill==3:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]263 main.ONOS3.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]264 elif kill==4:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]265 main.ONOS4.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]266 else:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]267 main.ONOS5.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]268
269 kill2 = randint(1,5)
270 if kill2==kill:
271 if kill2==5:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]272 main.ONOS4.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]273 elif kill2==1:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]274 main.ONOS3.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]275 else:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]276 main.ONOS5.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]277 else:
278 if kill2==1:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]279 main.ONOS5.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]280 elif kill2==2:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]281 main.ONOS2.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]282 elif kill2==3:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]283 main.ONOS3.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]284 elif kill2==4:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]285 main.ONOS4.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]286 else:
Jon Hall64b28222014-08-11 11:33:38 -0700[diff] [blame]287 main.ONOS5.kill()
adminecb92652014-08-04 09:27:21 -0700[diff] [blame]288 time.sleep(10)
289 main.step("Get the Mastership of each switch")
290 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
291 global masterSwitchList1
292 masterSwitchList1 = stdout
293
294
295
296 def CASE6(self,main) :
297 import os
298 main.case("Running ONOS Constant State Tests")
299 main.step("Get the current In-Memory Topology on each ONOS Instance and Compare it to the Topology before component failure")
300
301 main.step("Get the Mastership of each switch and compare to the Mastership before component failure")
302 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
303 result = main.TRUE
304 for i in range(1,29):
305 switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
306 switchDPID = switchDPID[:2]+":"+switchDPID[2:4]+":"+switchDPID[4:6]+":"+switchDPID[6:8]+":"+switchDPID[8:10]+":"+switchDPID[10:12]+":"+switchDPID[12:14]+":"+switchDPID[14:]
307 master1 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)
308 master2 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout)
309 if main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)==main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout):
310 result = result and main.TRUE
311 else:
312 result = main.FALSE
313 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Mastership of Switches was not changed",onfail="MASTERSHIP OF SWITCHES HAS CHANGED!!!")
314 result1 = result
315
316 main.step("Get the High Level Intents and compare to before component failure")
317 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
318 changesInIntents=main.ONOS1.comp_intents(preIntents=highIntentList1,postIntents=stdout)
319 if not changesInIntents:
320 result = main.TRUE
321 else:
322 main.log.info("THERE WERE CHANGES TO THE HIGH LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
323 result = main.FALSE
324 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to High level Intents",onfail="CHANGES WERE MADE TO HIGH LEVEL INTENTS")
325 result2 = result
326
327 main.step("Get the Low level Intents and compare to before component failure")
328 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
329 changesInIntents=main.ONOS1.comp_low(preIntents=lowIntentList1,postIntents=stdout)
330 if not changesInIntents:
331 result = main.TRUE
332 else:
333 main.log.info("THERE WERE CHANGES TO THE LOW LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
334 result = main.FALSE
335 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to Low level Intents",onfail="CHANGES WERE MADE TO LOW LEVEL INTENTS")
336 result3 = result
337
338
339 main.step("Get the OF Table entries and compare to before component failure")
340 result = main.TRUE
341 flows2=[]
342 for i in range(27):
343 flows2.append(main.Mininet2.get_flowTable(sw="s"+str(i+1)))
344 result = result and main.Mininet2.flow_comp(flow1=flows[i], flow2=main.Mininet2.get_flowTable(sw="s"+str(i+1)))
345 if result == main.FALSE:
346 main.log.info("DIFFERENCES IN FLOW TABLES FOR SWITCH "+str(i))
347 break
348 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes in the flow tables",onfail="CHANGES IN THE FLOW TABLES!!")
349 result4 = result
350
351 main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
352 main.Mininet2.pingKill()
353 result = main.FALSE
354 for i in range(8,18):
355 result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
356 if result==main.TRUE:
357 main.log.info("LOSS IN THE PINGS!")
358 elif result == main.ERROR:
359 main.log.info("There are multiple mininet process running!!")
360 else:
361 main.log.info("No Loss in the pings!")
362 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
363 result5 = not result
364 result = result1 and result2 and result3 and result4 and result5
365 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Constant State Tests Passed", onfail="CONSTANT STATE TESTS FAILED!!")
366
367 def CASE7 (self,main):
368 main.case("Killing a link to Ensure that Link Discovery is Working Properly")
369 main.step("Start continuous pings")
370 main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
371 main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
372 main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
373 main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
374 main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
375 main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
376 main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
377 main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
378 main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
379 main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
380
381 main.step("Determine the current number of switches and links")
382 (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
383 links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
384 main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
385
386 main.step("Kill Link between s3 and s28")
387 main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
388 time.sleep(5)
389 result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,str(int(links)-2))
390 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link Down discovered properly",onfail="LINKS NOT DISCOVERED PROPERLY")
391 result1 = result
392 result = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
393
394 main.step("Check for loss in pings when Link is brought down")
395 main.Mininet2.pingKill()
396 result = main.FALSE
397 for i in range(8,18):
398 result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
399 if result==main.TRUE:
400 main.log.info("LOSS IN THE PINGS!")
401 elif result == main.ERROR:
402 main.log.info("There are multiple mininet process running!!")
403 else:
404 main.log.info("No Loss in the pings!")
405 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
406 result2 = result
407 result = result1 and not result2
408 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link failure is discovered correctly and no traffic is lost!",onfail="Link Discovery failed or traffic was dropped!!!")
409
410 def CASE8 (self, main) :
411 import time
412 main.case("Killing a switch to ensure switch discovery is working properly")
413 main.step("Start continuous pings")
414 main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
415 main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
416 main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
417 main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
418 main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
419 main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
420 main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
421 main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
422 main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
423 main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
424
425 main.step("Determine the current number of switches and links")
426 (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
427 links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
428 main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
429
430 main.step("Kill s28 ")
431 main.Mininet2.del_switch("s28")
432 time.sleep(31)
433 result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],str(int(active)-1),str(int(links)-4))
434 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
435
436 main.step("Add back s28")
437 main.Mininet2.add_switch("s28")
438 main.Mininet1.assign_sw_controller(sw="28",ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
439 main.Mininet1.assign_sw_controller(sw="28",count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
440 time.sleep(31)
441 result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,links)
442 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
443 result1=result
444
445 main.step("Checking for Traffic Loss")
446 main.Mininet2.pingKill()
447 result = main.FALSE
448 for i in range(8,18):
449 result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
450 if result==main.TRUE:
451 main.log.info("LOSS IN THE PINGS!")
452 elif result == main.ERROR:
453 main.log.info("There are multiple mininet process running!!")
454 else:
455 main.log.info("No Loss in the pings!")
456 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
457 result = not result and result1
458 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovered Correctly and No Loss of traffic",onfail="Switch discovery failed or there was loss of traffic")