blob: 31db5f9f4e2010fb3b1ec1b4746c8a08035f8782 [file] [log] [blame]
adminecb92652014-08-04 09:27:21 -07001
2class HATestZK2:
3
4 global topology
5 global masterSwitchList
6 global highIntentList
7 global lowIntentList
8 global flows
9 flows = []
10
11 def __init__(self) :
12 self.default = ''
13
14 '''
15 CASE1 is to close any existing instances of ONOS, clean out the
16 RAMCloud database, and start up ONOS instances.
17 '''
18 def CASE1(self,main) :
19 main.case("Initial Startup")
20 main.step("Stop ONOS")
21 main.ONOS1.stop_all()
22 main.ONOS2.stop_all()
23 main.ONOS3.stop_all()
24 main.ONOS4.stop_all()
25 main.ONOS5.stop_all()
26 main.ONOS1.stop_rest()
27 main.ONOS2.stop_rest()
28 main.ONOS3.stop_rest()
29 main.ONOS4.stop_rest()
30 main.ONOS5.stop_rest()
31 result = main.ONOS1.status() or main.ONOS2.status() \
32 or main.ONOS3.status() or main.ONOS4.status() or main.ONOS5.status()
33 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="ONOS stopped successfully",onfail="ONOS WAS NOT KILLED PROPERLY")
34 main.step("Startup Zookeeper")
35 main.ZK1.start()
36 main.ZK2.start()
37 main.ZK3.start()
38 main.ZK4.start()
39 main.ZK5.start()
40 result = main.ZK1.isup() and main.ZK2.isup()\
41 and main.ZK3.isup() and main.ZK4.isup() and main.ZK5.isup()
42 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Zookeeper started successfully",onfail="ZOOKEEPER FAILED TO START")
43 main.step("Cleaning RC Database and Starting All")
44 main.RC1.del_db()
45 main.RC2.del_db()
46 main.RC3.del_db()
47 main.RC4.del_db()
48 main.RC5.del_db()
49 main.ONOS1.start_all()
50 main.ONOS2.start_all()
51 main.ONOS3.start_all()
52 main.ONOS4.start_all()
53 main.ONOS5.start_all()
54 # main.ONOS1.start_rest()
55 main.step("Testing Startup")
56 result1 = main.ONOS1.rest_status()
57 vm1 = main.RC1.status_coor and main.RC1.status_serv and \
58 main.ONOS1.isup()
59 vm2 = main.RC2.status_coor and main.ONOS2.isup()
60 vm3 = main.RC3.status_coor and main.ONOS3.isup()
61 vm4 = main.RC4.status_coor and main.ONOS4.isup()
62 vm5 = main.RC5.status_coor and main.ONOS5.isup()
63 result = result1 and vm1 and vm2 and vm3 and vm4 and vm5
64 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Everything started successfully",onfail="EVERYTHING FAILED TO START")
65 if result==main.FALSE:
66 main.cleanup()
67 main.exit()
68
69 '''
70 CASE2
71 '''
72 def CASE2(self,main) :
73 import time
74 import json
75 import re
76 main.log.report("Assigning Controllers")
77 main.case("Assigning Controllers")
78 main.step("Assign Master Controllers")
79 for i in range(1,29):
80 if i ==1:
81 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
82 elif i>=2 and i<5:
83 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip2'],port1=main.params['CTRL']['port2'])
84 elif i>=5 and i<8:
85 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip3'],port1=main.params['CTRL']['port3'])
86 elif i>=8 and i<18:
87 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip4'],port1=main.params['CTRL']['port4'])
88 elif i>=18 and i<28:
89 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip5'],port1=main.params['CTRL']['port5'])
90 else:
91 main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
92
93 result = main.TRUE
94 for i in range (1,29):
95 if i==1:
96 response = main.Mininet1.get_sw_controller("s"+str(i))
97 print("Response is " + str(response))
98 if re.search("tcp:"+main.params['CTRL']['ip1'],response):
99 result = result and main.TRUE
100 else:
101 result = main.FALSE
102 elif i>=2 and i<5:
103 response = main.Mininet1.get_sw_controller("s"+str(i))
104 print("Response is " + str(response))
105 if re.search("tcp:"+main.params['CTRL']['ip2'],response):
106 result = result and main.TRUE
107 else:
108 result = main.FALSE
109 elif i>=5 and i<8:
110 response = main.Mininet1.get_sw_controller("s"+str(i))
111 print("Response is " + str(response))
112 if re.search("tcp:"+main.params['CTRL']['ip3'],response):
113 result = result and main.TRUE
114 else:
115 result = main.FALSE
116 elif i>=8 and i<18:
117 response = main.Mininet1.get_sw_controller("s"+str(i))
118 print("Response is " + str(response))
119 if re.search("tcp:"+main.params['CTRL']['ip4'],response):
120 result = result and main.TRUE
121 else:
122 result = main.FALSE
123 elif i>=18 and i<28:
124 response = main.Mininet1.get_sw_controller("s"+str(i))
125 print("Response is " + str(response))
126 if re.search("tcp:"+main.params['CTRL']['ip5'],response):
127 result = result and main.TRUE
128 else:
129 result = main.FALSE
130 else:
131 response = main.Mininet1.get_sw_controller("s"+str(i))
132 print("Response is" + str(response))
133 if re.search("tcp:" +main.params['CTRL']['ip1'],response):
134 result = result and main.TRUE
135 else:
136 result = main.FALSE
137
138 utilities.assert_equals(expect = main.TRUE,actual=result,onpass="MasterControllers assigned correctly")
139 for i in range (1,29):
140 main.Mininet1.assign_sw_controller(sw=str(i),count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
141
142 def CASE3(self,main) :
143 import time
144 import json
145 import re
146 main.case("Adding Intents")
147 intentIP = main.params['CTRL']['ip1']
148 intentPort=main.params['INTENTS']['intentPort']
149 intentURL=main.params['INTENTS']['intentURL']
150 count = 1
151 for i in range(8,18):
152 srcMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
153 dstMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
154 srcDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
155 dstDPID= '00:00:00:00:00:00:60:' +str(i+10)
156 main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
157 count+=1
158 dstDPID = '00:00:00:00:00:00:30:'+str(i).zfill(2)
159 srcDPID= '00:00:00:00:00:00:60:' +str(i+10)
160 dstMac = '00:00:00:00:00:' + str(hex(i)[2:]).zfill(2)
161 srcMac = '00:00:00:00:00:'+str(hex(i+10)[2:])
162 main.ONOS1.add_intent(intent_id=str(count),src_dpid=srcDPID,dst_dpid=dstDPID,src_mac=srcMac,dst_mac=dstMac,intentIP=intentIP,intentPort=intentPort,intentURL=intentURL)
163 count+=1
164 count = 1
165 i = 8
166 result = main.TRUE
167 while i <18 :
168 main.log.info("\n\nh"+str(i)+" is Pinging h" + str(i+10))
169 ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
170 if ping ==main.FALSE and count <9:
171 count+=1
172 i = 8
173 result = main.FALSE
174 main.log.info("Ping FAILED! Making attempt number "+str(count) + "in 2 seconds")
175 time.sleep(2)
176 elif ping==main.FALSE:
177 main.log.info("PINGS FAILED! MAX RETRIES REACHED!")
178 i=19
179 result = main.FALSE
180 elif ping==main.TRUE:
181 main.log.info("Ping passed!")
182 i+=1
183 result = main.TRUE
184 else:
185 main.log.info("ERROR!!")
186 result = main.ERROR
187 if result==main.FALSE:
188 main.log.info("INTENTS HAVE NOT BEEN INSTALLED CORRECTLY!! EXITING!!!")
189 main.cleanup()
190 main.exit()
191
192
193 def CASE4(self,main) :
194 import time
195 from subprocess import Popen, PIPE
196 main.case("Setting up and Gathering data for current state")
197 main.step("Get the current In-Memory Topology on each ONOS Instance")
198
199 '''
200 ctrls = []
201 count = 1
202 while True:
203 temp = ()
204 if ('ip'+str(count)) in main.params['CTRL']:
205 temp = temp+(getattr(main,('ONOS'+str(count))),)
206 temp = temp + ("ONOS"+str(count),)
207 temp = temp + (main.params['CTRL']['ip'+str(count)],)
208 temp = temp + (eval(main.params['CTRL']['port'+str(count)]),)
209 ctrls.append(temp)
210 count+=1
211 else:
212 break
213 topo_result = main.TRUE
214
215 for n in range(1,count):
216 temp_result = main.Mininet1.compare_topo(ctrls,main.ONOS1.get_json(main.params['CTRL']['ip'+str(n)]+":"+main.params['CTRL']['restPort'+str(n)]+main.params['TopoRest']))
217 '''
218
219 main.step("Get the Mastership of each switch")
220 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
221 global masterSwitchList1
222 masterSwitchList1 = stdout
223
224 main.step("Get the High Level Intents")
225 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
226 global highIntentList1
227 highIntentList1 = stdout
228
229 main.step("Get the Low level Intents")
230 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
231 global lowIntentList1
232 lowIntentList1= stdout
233
234 main.step("Get the OF Table entries")
235 global flows
236 flows=[]
237 for i in range(1,29):
238 flows.append(main.Mininet2.get_flowTable("s"+str(i)))
239
240
241 main.step("Start continuous pings")
242 main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
243 main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
244 main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
245 main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
246 main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
247 main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
248 main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
249 main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
250 main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
251 main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
252
253
254 def CASE5(self,main) :
255 import re
256 main.case("MAIN COMPONENT FAILURE AND SCENARIO SPECIFIC TESTS")
257 main.step("Determine the current number of switches and links")
258 (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
259 links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
260
261 main.step("Zookeeper Server Failure!")
262 main.ZK1.stop()
263 main.ZK2.stop()
264 main.ZK3.stop()
265 main.ZK4.stop()
266 main.ZK5.stop()
267 main.Mininet2.del_switch("s1")
268 time.sleep(31)
269 result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],str(int(active)-1),str(int(links)-2))
270 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="Registry is no longer active",onfail="Registry is still being updated")
271
272 main.step("Restart Zookeeper")
273 main.ZK1.start()
274 main.ZK2.start()
275 main.ZK3.start()
276 main.ZK4.start()
277 main.ZK5.start()
278 time.sleep(10) # Time for Zookeeper to reboot
279 master1=main.ZK1.status()
280 master2=main.ZK2.status()
281 master3=main.ZK3.status()
282 master4=main.ZK4.status()
283 master5=main.ZK5.status()
284 if re.search("leader",master1) or re.search("leader",master2) or re.search("leader",master3) or re.search("leader",master4) or re.search("leader",master5):
285 main.log.info("New ZK Leader Elected")
286 else:
287 main.log.info("NO NEW ZK LEADER ELECTED!!!")
288 main.step("Add back s1")
289 main.Mininet2.add_switch("s1")
290 main.Mininet1.assign_sw_controller(sw="1",ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
291 main.Mininet1.assign_sw_controller(sw="1",count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
292 time.sleep(31)
293
294
295 def CASE6(self,main) :
296 import os
297 main.case("Running ONOS Constant State Tests")
298 main.step("Get the current In-Memory Topology on each ONOS Instance and Compare it to the Topology before component failure")
299
300 main.step("Get the Mastership of each switch and compare to the Mastership before component failure")
301 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['switchURL']],stdout=PIPE).communicate()
302 result = main.TRUE
303 for i in range(1,29):
304 switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i)))
305 switchDPID = switchDPID[:2]+":"+switchDPID[2:4]+":"+switchDPID[4:6]+":"+switchDPID[6:8]+":"+switchDPID[8:10]+":"+switchDPID[10:12]+":"+switchDPID[12:14]+":"+switchDPID[14:]
306 master1 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)
307 master2 = main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout)
308 if main.ZK1.findMaster(switchDPID=switchDPID,switchList=masterSwitchList1)==main.ZK1.findMaster(switchDPID=switchDPID,switchList=stdout):
309 result = result and main.TRUE
310 else:
311 result = main.FALSE
312 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Mastership of Switches was not changed",onfail="MASTERSHIP OF SWITCHES HAS CHANGED!!!")
313 result1 = result
314
315 main.step("Get the High Level Intents and compare to before component failure")
316 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentHighURL']],stdout=PIPE).communicate()
317 changesInIntents=main.ONOS1.comp_intents(preIntents=highIntentList1,postIntents=stdout)
318 if not changesInIntents:
319 result = main.TRUE
320 else:
321 main.log.info("THERE WERE CHANGES TO THE HIGH LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
322 result = main.FALSE
323 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to High level Intents",onfail="CHANGES WERE MADE TO HIGH LEVEL INTENTS")
324 result2=result
325
326 main.step("Get the Low level Intents and compare to before component failure")
327 (stdout,stderr)=Popen(["curl",main.params['CTRL']['ip1']+":"+main.params['CTRL']['restPort1']+main.params['CTRL']['intentLowURL']],stdout=PIPE).communicate()
328 changesInIntents=main.ONOS1.comp_low(preIntents=lowIntentList1,postIntents=stdout)
329 if not changesInIntents:
330 result = main.TRUE
331 else:
332 main.log.info("THERE WERE CHANGES TO THE LOW LEVEL INTENTS! CHANGES WERE: "+str(changesInIntents))
333 result = main.FALSE
334 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes to Low level Intents",onfail="CHANGES WERE MADE TO LOW LEVEL INTENTS")
335 result3=result
336
337
338 main.step("Get the OF Table entries and compare to before component failure")
339 result = main.TRUE
340 flows2=[]
341 for i in range(27):
342 flows2.append(main.Mininet2.get_flowTable(sw="s"+str(i+1)))
343 result = result and main.Mininet2.flow_comp(flow1=flows[i], flow2=main.Mininet2.get_flowTable(sw="s"+str(i+1)))
344 if result == main.FALSE:
345 main.log.info("DIFFERENCES IN FLOW TABLES FOR SWITCH "+str(i))
346 break
347 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No changes in the flow tables",onfail="CHANGES IN THE FLOW TABLES!!")
348 result4 = result
349
350 main.step("Check the continuous pings to ensure that no packets were dropped during component failure")
351 main.Mininet2.pingKill()
352 result = main.FALSE
353 for i in range(8,18):
354 result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
355 if result==main.TRUE:
356 main.log.info("LOSS IN THE PINGS!")
357 elif result == main.ERROR:
358 main.log.info("There are multiple mininet process running!!")
359 else:
360 main.log.info("No Loss in the pings!")
361 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
362 result5=not result
363 result = result1 and result2 and result3 and result4 and result5
364 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Constant State Tests Passed!",onfail="CONSTANT STATE TESTS FAILED!!")
365
366 def CASE7 (self,main):
367 main.case("Killing a link to Ensure that Link Discovery is Working Properly")
368 main.step("Start continuous pings")
369 main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
370 main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
371 main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
372 main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
373 main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
374 main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
375 main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
376 main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
377 main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
378 main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
379
380 main.step("Determine the current number of switches and links")
381 (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
382 links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
383 main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
384
385 main.step("Kill Link between s3 and s28")
386 main.Mininet1.link(END1="s3",END2="s28",OPTION="down")
387 result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,str(int(links)-2))
388 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link Down discovered properly",onfail="LINKS NOT DISCOVERED PROPERLY")
389 result1 = result
390 result = main.Mininet1.link(END1="s3",END2="s28",OPTION="up")
391
392 main.step("Check for loss in pings when Link is brought down")
393 main.Mininet2.pingKill()
394 result = main.FALSE
395 for i in range(8,18):
396 result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
397 if result==main.TRUE:
398 main.log.info("LOSS IN THE PINGS!")
399 elif result == main.ERROR:
400 main.log.info("There are multiple mininet process running!!")
401 else:
402 main.log.info("No Loss in the pings!")
403 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
404 result2 = result
405 result = result1 and not result2
406 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="Link failure is discovered correctly and no traffic is lost!",onfail="Link Discovery failed or traffic was dropped!!!")
407
408
409
410 def CASE8 (self, main) :
411 import time
412 main.case("Killing a switch to ensure switch discovery is working properly")
413 main.step("Start continuous pings")
414 main.Mininet2.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'],pingTime=500)
415 main.Mininet2.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'],pingTime=500)
416 main.Mininet2.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'],pingTime=500)
417 main.Mininet2.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'],pingTime=500)
418 main.Mininet2.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'],pingTime=500)
419 main.Mininet2.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'],pingTime=500)
420 main.Mininet2.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'],pingTime=500)
421 main.Mininet2.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'],pingTime=500)
422 main.Mininet2.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'],pingTime=500)
423 main.Mininet2.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'],pingTime=500)
424
425 main.step("Determine the current number of switches and links")
426 (number,active)=main.ONOS1.num_switch(RestIP=main.params['CTRL']['ip1'])
427 links = main.ONOS1.num_link(RestIP=main.params['CTRL']['ip1'])
428 main.log.info("Currently there are %s switches, %s are active, and %s links" %(number,active,links))
429
430 main.step("Kill s28 ")
431 main.Mininet2.del_switch("s28")
432 time.sleep(31)
433 result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],str(int(active)-1),str(int(links)-4))
434 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
435
436 main.step("Add back s28")
437 main.Mininet2.add_switch("s28")
438 main.Mininet1.assign_sw_controller(sw="28",ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
439 main.Mininet1.assign_sw_controller(sw="28",count=5,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'],ip5=main.params['CTRL']['ip5'],port5=main.params['CTRL']['port5'])
440 time.sleep(31)
441 result = main.ONOS1.check_status_report(main.params['CTRL']['ip1'],active,links)
442 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovery is Working",onfail="Switch Discovery FAILED TO WORK PROPERLY!")
443 result1=result
444
445 main.step("Checking for Traffic Loss")
446 main.Mininet2.pingKill()
447 result = main.FALSE
448 for i in range(8,18):
449 result = result or main.Mininet2.checkForLoss("/tmp/ping.h"+str(i))
450 if result==main.TRUE:
451 main.log.info("LOSS IN THE PINGS!")
452 elif result == main.ERROR:
453 main.log.info("There are multiple mininet process running!!")
454 else:
455 main.log.info("No Loss in the pings!")
456 utilities.assert_equals(expect=main.FALSE,actual=result,onpass="No Loss of connectivity!",onfail="LOSS OF CONNECTIVITY")
457 result = not result and result1
458 utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Switch Discovered Correctly and No Loss of traffic",onfail="Switch discovery failed or there was loss of traffic")
459