blob: 58d5f349cd1a7206e8ad0e7b88662dad95d296d7 [file] [log] [blame]
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001"""
2 Wrapper functions for FuncIntent
3 This functions include Onosclidriver and Mininetclidriver driver functions
4 Author: kelvin@onlab.us
5"""
6import time
7import copy
8import json
9
10def __init__( self ):
11 self.default = ''
12
13def hostIntent( main,
14 name,
15 host1,
16 host2,
17 onosNode=0,
18 host1Id="",
19 host2Id="",
20 mac1="",
21 mac2="",
22 vlan1="-1",
23 vlan2="-1",
24 sw1="",
25 sw2="",
26 expectedLink=0 ):
27 """
kelvin-onlab58dc39e2015-08-06 08:11:09 -070028 Description:
29 Verify add-host-intent
30 Steps:
31 - Discover hosts
32 - Add host intents
33 - Check intents
34 - Verify flows
35 - Ping hosts
36 - Reroute
37 - Link down
kelvin-onlabd48a68c2015-07-13 16:01:36 -070038 - Verify flows
kelvin-onlab58dc39e2015-08-06 08:11:09 -070039 - Check topology
kelvin-onlabd48a68c2015-07-13 16:01:36 -070040 - Ping hosts
kelvin-onlab58dc39e2015-08-06 08:11:09 -070041 - Link up
42 - Verify flows
43 - Check topology
44 - Ping hosts
45 - Remove intents
46 Required:
47 name - Type of host intent to add eg. IPV4 | VLAN | Dualstack
48 host1 - Name of first host
49 host2 - Name of second host
50 Optional:
51 onosNode - ONOS node to install the intents in main.CLIs[ ]
52 0 by default so that it will always use the first
53 ONOS node
54 host1Id - ONOS id of the first host eg. 00:00:00:00:00:01/-1
55 host2Id - ONOS id of the second host
56 mac1 - Mac address of first host
57 mac2 - Mac address of the second host
58 vlan1 - Vlan tag of first host, defaults to -1
59 vlan2 - Vlan tag of second host, defaults to -1
60 sw1 - First switch to bring down & up for rerouting purpose
61 sw2 - Second switch to bring down & up for rerouting purpose
62 expectedLink - Expected link when the switches are down, it should
63 be two links lower than the links before the two
64 switches are down
65 Return:
66 Returns main.TRUE if all verification passed, otherwise return
67 main.FALSE; returns main.FALSE if there is a key error
kelvin-onlabd48a68c2015-07-13 16:01:36 -070068 """
69
70 # Assert variables
71 assert main, "There is no main variable"
72 assert name, "variable name is empty"
73 assert host1 and host2, "You must specify hosts"
74
75 global itemName
76 itemName = name
77 h1Id = host1Id
78 h2Id = host2Id
79 h1Mac = mac1
80 h2Mac = mac2
81 vlan1 = vlan1
82 vlan2 = vlan2
83 hostNames = [ host1 , host2 ]
84 intentsId = []
85 stepResult = main.TRUE
86 pingResult = main.TRUE
87 intentResult = main.TRUE
88 removeIntentResult = main.TRUE
89 flowResult = main.TRUE
90 topoResult = main.TRUE
91 linkDownResult = main.TRUE
92 linkUpResult = main.TRUE
93 onosNode = int( onosNode )
94
kelvin-onlab6dea6e62015-07-23 13:07:26 -070095 try:
96 if main.hostsData:
97 if not h1Mac:
98 h1Mac = main.hostsData[ host1 ][ 'mac' ]
99 if not h2Mac:
100 h2Mac = main.hostsData[ host2 ][ 'mac' ]
101 if main.hostsData[ host1 ].get( 'vlan' ):
102 vlan1 = main.hostsData[ host1 ][ 'vlan' ]
103 if main.hostsData[ host1 ].get( 'vlan' ):
104 vlan2 = main.hostsData[ host2 ][ 'vlan' ]
105 if not h1Id:
106 h1Id = main.hostsData[ host1 ][ 'id' ]
107 if not h2Id:
108 h2Id = main.hostsData[ host2 ][ 'id' ]
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700109
kelvin-onlab6dea6e62015-07-23 13:07:26 -0700110 assert h1Id and h2Id, "You must specify host IDs"
111 if not ( h1Id and h2Id ):
112 main.log.info( "There are no host IDs" )
113 return main.FALSE
114
kelvin-onlab6dea6e62015-07-23 13:07:26 -0700115 except KeyError:
116 main.log.error( itemName + ": Key error Exception" )
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700117 return main.FALSE
118
kelvin-onlab0ad05d12015-07-23 14:21:15 -0700119 # Discover hosts using arping incase pingall discovery failed
120 main.log.info( itemName + ": Discover host using arping" )
121 main.Mininet1.arping( host=host1 )
122 main.Mininet1.arping( host=host2 )
123 host1 = main.CLIs[ 0 ].getHost( mac=h1Mac )
124 host2 = main.CLIs[ 0 ].getHost( mac=h2Mac )
125
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700126 # Check flows count in each node
127 checkFlowsCount( main )
128
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700129 # Adding host intents
130 main.log.info( itemName + ": Adding host intents" )
131 intent1 = main.CLIs[ onosNode ].addHostIntent( hostIdOne=h1Id,
kelvin-onlab6dea6e62015-07-23 13:07:26 -0700132 hostIdTwo=h2Id )
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700133 intentsId.append( intent1 )
134
135 # Check intents state
136 time.sleep( main.checkIntentSleep )
137 intentResult = checkIntentState( main, intentsId )
138 checkFlowsCount( main )
139
140 # Check intents state again if first check fails...
141 if not intentResult:
142 intentResult = checkIntentState( main, intentsId )
143
144 # Check flows count in each node
145 checkFlowsCount( main )
146 # Verify flows
147 checkFlowsState( main )
148
149 # Ping hosts
150 firstPingResult = pingallHosts( main, hostNames )
151 if not firstPingResult:
152 main.log.debug( "First ping failed, there must be" +
153 " something wrong with ONOS performance" )
154
155 # Ping hosts again...
156 pingResult = pingResult and pingallHosts( main, hostNames )
157
158 # Test rerouting if these variables exist
159 if sw1 and sw2 and expectedLink:
160 # link down
161 linkDownResult = link( main, sw1, sw2, "down" )
162 intentResult = intentResult and checkIntentState( main, intentsId )
163
164 # Check flows count in each node
165 checkFlowsCount( main )
166 # Verify flows
167 checkFlowsState( main )
168
169 # Check OnosTopology
170 topoResult = checkTopology( main, expectedLink )
171
172 # Ping hosts
173 pingResult = pingResult and pingallHosts( main, hostNames )
174
175 intentResult = checkIntentState( main, intentsId )
176
177 # Checks ONOS state in link down
178 if linkDownResult and topoResult and pingResult and intentResult:
179 main.log.info( itemName + ": Successfully brought link down" )
180 else:
181 main.log.error( itemName + ": Failed to bring link down" )
182
183 # link up
184 linkUpResult = link( main, sw1, sw2, "up" )
185 time.sleep( main.rerouteSleep )
186
187 # Check flows count in each node
188 checkFlowsCount( main )
189 # Verify flows
190 checkFlowsState( main )
191
192 # Check OnosTopology
193 topoResult = checkTopology( main, main.numLinks )
194
195 # Ping hosts
196 pingResult = pingResult and pingallHosts( main, hostNames )
197
198 intentResult = checkIntentState( main, intentsId )
199
200 # Checks ONOS state in link up
201 if linkUpResult and topoResult and pingResult and intentResult:
202 main.log.info( itemName + ": Successfully brought link back up" )
203 else:
204 main.log.error( itemName + ": Failed to bring link back up" )
205
206 # Remove all intents
207 removeIntentResult = removeAllIntents( main, intentsId )
208
209 stepResult = pingResult and linkDownResult and linkUpResult \
210 and intentResult and removeIntentResult
211
212 return stepResult
213
214def pointIntent( main,
215 name,
216 host1,
217 host2,
218 onosNode=0,
219 deviceId1="",
220 deviceId2="",
221 port1="",
222 port2="",
223 ethType="",
224 mac1="",
225 mac2="",
226 bandwidth="",
227 lambdaAlloc=False,
228 ipProto="",
229 ip1="",
230 ip2="",
231 tcp1="",
232 tcp2="",
233 sw1="",
234 sw2="",
235 expectedLink=0 ):
236
237 """
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700238 Description:
239 Verify add-point-intent
240 Steps:
241 - Get device ids | ports
242 - Add point intents
243 - Check intents
244 - Verify flows
245 - Ping hosts
246 - Reroute
247 - Link down
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700248 - Verify flows
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700249 - Check topology
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700250 - Ping hosts
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700251 - Link up
252 - Verify flows
253 - Check topology
254 - Ping hosts
255 - Remove intents
256 Required:
257 name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
258 host1 - Name of first host
259 host2 - Name of second host
260 Optional:
261 onosNode - ONOS node to install the intents in main.CLIs[ ]
262 0 by default so that it will always use the first
263 ONOS node
264 deviceId1 - ONOS device id of the first switch, the same as the
265 location of the first host eg. of:0000000000000001/1,
266 located at device 1 port 1
267 deviceId2 - ONOS device id of the second switch
268 port1 - The port number where the first host is attached
269 port2 - The port number where the second host is attached
270 ethType - Ethernet type eg. IPV4, IPV6
271 mac1 - Mac address of first host
272 mac2 - Mac address of the second host
273 bandwidth - Bandwidth capacity
274 lambdaAlloc - Allocate lambda, defaults to False
275 ipProto - IP protocol
276 ip1 - IP address of first host
277 ip2 - IP address of second host
278 tcp1 - TCP port of first host
279 tcp2 - TCP port of second host
280 sw1 - First switch to bring down & up for rerouting purpose
281 sw2 - Second switch to bring down & up for rerouting purpose
282 expectedLink - Expected link when the switches are down, it should
283 be two links lower than the links before the two
284 switches are down
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700285 """
286
287 assert main, "There is no main variable"
288 assert name, "variable name is empty"
289 assert host1 and host2, "You must specify hosts"
290
291 global itemName
292 itemName = name
293 host1 = host1
294 host2 = host2
295 hostNames = [ host1, host2 ]
296 intentsId = []
297
298 pingResult = main.TRUE
299 intentResult = main.TRUE
300 removeIntentResult = main.TRUE
301 flowResult = main.TRUE
302 topoResult = main.TRUE
303 linkDownResult = main.TRUE
304 linkUpResult = main.TRUE
305 onosNode = int( onosNode )
306
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700307 # Adding bidirectional point intents
308 main.log.info( itemName + ": Adding point intents" )
309 intent1 = main.CLIs[ onosNode ].addPointIntent( ingressDevice=deviceId1,
310 egressDevice=deviceId2,
311 portIngress=port1,
312 portEgress=port2,
313 ethType=ethType,
314 ethSrc=mac1,
315 ethDst=mac2,
316 bandwidth=bandwidth,
317 lambdaAlloc=lambdaAlloc,
318 ipProto=ipProto,
319 ipSrc=ip1,
320 ipDst=ip2,
321 tcpSrc=tcp1,
322 tcpDst=tcp2 )
323
324 intentsId.append( intent1 )
325 intent2 = main.CLIs[ onosNode ].addPointIntent( ingressDevice=deviceId2,
326 egressDevice=deviceId1,
327 portIngress=port2,
328 portEgress=port1,
329 ethType=ethType,
330 ethSrc=mac2,
331 ethDst=mac1,
332 bandwidth=bandwidth,
333 lambdaAlloc=lambdaAlloc,
334 ipProto=ipProto,
335 ipSrc=ip2,
336 ipDst=ip1,
337 tcpSrc=tcp2,
338 tcpDst=tcp1 )
339 intentsId.append( intent2 )
340
341 # Check intents state
342 time.sleep( main.checkIntentSleep )
343 intentResult = checkIntentState( main, intentsId )
344 # Check flows count in each node
345 checkFlowsCount( main )
346
347 # Check intents state again if first check fails...
348 if not intentResult:
349 intentResult = checkIntentState( main, intentsId )
350
351 # Check flows count in each node
352 checkFlowsCount( main )
353 # Verify flows
354 checkFlowsState( main )
355
356 # Ping hosts
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700357 pingResult = pingResult and pingallHosts( main, hostNames )
358
359 # Test rerouting if these variables exist
360 if sw1 and sw2 and expectedLink:
361 # link down
362 linkDownResult = link( main, sw1, sw2, "down" )
363 intentResult = intentResult and checkIntentState( main, intentsId )
364
365 # Check flows count in each node
366 checkFlowsCount( main )
367 # Verify flows
368 checkFlowsState( main )
369
370 # Check OnosTopology
371 topoResult = checkTopology( main, expectedLink )
372
373 # Ping hosts
374 pingResult = pingResult and pingallHosts( main, hostNames )
375
376 intentResult = checkIntentState( main, intentsId )
377
378 # Checks ONOS state in link down
379 if linkDownResult and topoResult and pingResult and intentResult:
380 main.log.info( itemName + ": Successfully brought link down" )
381 else:
382 main.log.error( itemName + ": Failed to bring link down" )
383
384 # link up
385 linkUpResult = link( main, sw1, sw2, "up" )
386 time.sleep( main.rerouteSleep )
387
388 # Check flows count in each node
389 checkFlowsCount( main )
390 # Verify flows
391 checkFlowsState( main )
392
393 # Check OnosTopology
394 topoResult = checkTopology( main, main.numLinks )
395
396 # Ping hosts
397 pingResult = pingResult and pingallHosts( main, hostNames )
398
399 intentResult = checkIntentState( main, intentsId )
400
401 # Checks ONOS state in link up
402 if linkUpResult and topoResult and pingResult and intentResult:
403 main.log.info( itemName + ": Successfully brought link back up" )
404 else:
405 main.log.error( itemName + ": Failed to bring link back up" )
406
407 # Remove all intents
408 removeIntentResult = removeAllIntents( main, intentsId )
409
410 stepResult = pingResult and linkDownResult and linkUpResult \
411 and intentResult and removeIntentResult
412
413 return stepResult
414
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700415def pointIntentTcp( main,
416 name,
417 host1,
418 host2,
419 onosNode=0,
420 deviceId1="",
421 deviceId2="",
422 port1="",
423 port2="",
424 ethType="",
425 mac1="",
426 mac2="",
427 bandwidth="",
428 lambdaAlloc=False,
429 ipProto="",
430 ip1="",
431 ip2="",
432 tcp1="",
433 tcp2="",
434 sw1="",
435 sw2="",
436 expectedLink=0 ):
437
438 """
439 Description:
440 Verify add-point-intent only for TCP
441 Steps:
442 - Get device ids | ports
443 - Add point intents
444 - Check intents
445 - Verify flows
446 - Ping hosts
447 - Reroute
448 - Link down
449 - Verify flows
450 - Check topology
451 - Ping hosts
452 - Link up
453 - Verify flows
454 - Check topology
455 - Ping hosts
456 - Remove intents
457 Required:
458 name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
459 host1 - Name of first host
460 host2 - Name of second host
461 Optional:
462 onosNode - ONOS node to install the intents in main.CLIs[ ]
463 0 by default so that it will always use the first
464 ONOS node
465 deviceId1 - ONOS device id of the first switch, the same as the
466 location of the first host eg. of:0000000000000001/1,
467 located at device 1 port 1
468 deviceId2 - ONOS device id of the second switch
469 port1 - The port number where the first host is attached
470 port2 - The port number where the second host is attached
471 ethType - Ethernet type eg. IPV4, IPV6
472 mac1 - Mac address of first host
473 mac2 - Mac address of the second host
474 bandwidth - Bandwidth capacity
475 lambdaAlloc - Allocate lambda, defaults to False
476 ipProto - IP protocol
477 ip1 - IP address of first host
478 ip2 - IP address of second host
479 tcp1 - TCP port of first host
480 tcp2 - TCP port of second host
481 sw1 - First switch to bring down & up for rerouting purpose
482 sw2 - Second switch to bring down & up for rerouting purpose
483 expectedLink - Expected link when the switches are down, it should
484 be two links lower than the links before the two
485 switches are down
486 """
487
488 assert main, "There is no main variable"
489 assert name, "variable name is empty"
490 assert host1 and host2, "You must specify hosts"
491
492 global itemName
493 itemName = name
494 host1 = host1
495 host2 = host2
496 hostNames = [ host1, host2 ]
497 intentsId = []
498
499 iperfResult = main.TRUE
500 intentResult = main.TRUE
501 removeIntentResult = main.TRUE
502 flowResult = main.TRUE
503 topoResult = main.TRUE
504 linkDownResult = main.TRUE
505 linkUpResult = main.TRUE
506 onosNode = int( onosNode )
507
508 # Adding bidirectional point intents
509 main.log.info( itemName + ": Adding point intents" )
510 intent1 = main.CLIs[ onosNode ].addPointIntent( ingressDevice=deviceId1,
511 egressDevice=deviceId2,
512 portIngress=port1,
513 portEgress=port2,
514 ethType=ethType,
515 ethSrc=mac1,
516 ethDst=mac2,
517 bandwidth=bandwidth,
518 lambdaAlloc=lambdaAlloc,
519 ipProto=ipProto,
520 ipSrc=ip1,
521 ipDst=ip2,
522 tcpSrc=tcp1,
523 tcpDst="" )
524
525 intent2 = main.CLIs[ onosNode ].addPointIntent( ingressDevice=deviceId2,
526 egressDevice=deviceId1,
527 portIngress=port2,
528 portEgress=port1,
529 ethType=ethType,
530 ethSrc=mac2,
531 ethDst=mac1,
532 bandwidth=bandwidth,
533 lambdaAlloc=lambdaAlloc,
534 ipProto=ipProto,
535 ipSrc=ip2,
536 ipDst=ip1,
537 tcpSrc=tcp2,
538 tcpDst="" )
539
540 intent3 = main.CLIs[ onosNode ].addPointIntent( ingressDevice=deviceId1,
541 egressDevice=deviceId2,
542 portIngress=port1,
543 portEgress=port2,
544 ethType=ethType,
545 ethSrc=mac1,
546 ethDst=mac2,
547 bandwidth=bandwidth,
548 lambdaAlloc=lambdaAlloc,
549 ipProto=ipProto,
550 ipSrc=ip1,
551 ipDst=ip2,
552 tcpSrc="",
553 tcpDst=tcp2 )
554
555 intent4 = main.CLIs[ onosNode ].addPointIntent( ingressDevice=deviceId2,
556 egressDevice=deviceId1,
557 portIngress=port2,
558 portEgress=port1,
559 ethType=ethType,
560 ethSrc=mac2,
561 ethDst=mac1,
562 bandwidth=bandwidth,
563 lambdaAlloc=lambdaAlloc,
564 ipProto=ipProto,
565 ipSrc=ip2,
566 ipDst=ip1,
567 tcpSrc="",
568 tcpDst=tcp1 )
569 intentsId.append( intent1 )
570 intentsId.append( intent2 )
571 intentsId.append( intent3 )
572 intentsId.append( intent4 )
573
574 # Check intents state
575 time.sleep( main.checkIntentSleep )
576 intentResult = checkIntentState( main, intentsId )
577 # Check flows count in each node
578 checkFlowsCount( main )
579
580 # Check intents state again if first check fails...
581 if not intentResult:
582 intentResult = checkIntentState( main, intentsId )
583
584 # Check flows count in each node
585 checkFlowsCount( main )
586
587 # Verify flows
588 checkFlowsState( main )
589
590 # Run iperf to both host
591 iperfResult = iperfResult and main.Mininet1.iperftcp( host1,
592 host2, 10 )
593
594 # Test rerouting if these variables exist
595 if sw1 and sw2 and expectedLink:
596 # link down
597 linkDownResult = link( main, sw1, sw2, "down" )
598 intentResult = intentResult and checkIntentState( main, intentsId )
599
600 # Check flows count in each node
601 checkFlowsCount( main )
602 # Verify flows
603 checkFlowsState( main )
604
605 # Check OnosTopology
606 topoResult = checkTopology( main, expectedLink )
607
608 # Run iperf to both host
609 iperfResult = iperfResult and main.Mininet1.iperftcp( host1,
610 host2, 10 )
611
612 intentResult = checkIntentState( main, intentsId )
613
614 # Checks ONOS state in link down
615 if linkDownResult and topoResult and iperfResult and intentResult:
616 main.log.info( itemName + ": Successfully brought link down" )
617 else:
618 main.log.error( itemName + ": Failed to bring link down" )
619
620 # link up
621 linkUpResult = link( main, sw1, sw2, "up" )
622 time.sleep( main.rerouteSleep )
623
624 # Check flows count in each node
625 checkFlowsCount( main )
626 # Verify flows
627 checkFlowsState( main )
628
629 # Check OnosTopology
630 topoResult = checkTopology( main, main.numLinks )
631
632 # Run iperf to both host
633 iperfResult = iperfResult and main.Mininet1.iperftcp( host1,
634 host2, 10 )
635
636 intentResult = checkIntentState( main, intentsId )
637
638 # Checks ONOS state in link up
639 if linkUpResult and topoResult and iperfResult and intentResult:
640 main.log.info( itemName + ": Successfully brought link back up" )
641 else:
642 main.log.error( itemName + ": Failed to bring link back up" )
643
644 # Remove all intents
645 removeIntentResult = removeAllIntents( main, intentsId )
646
647 stepResult = iperfResult and linkDownResult and linkUpResult \
648 and intentResult and removeIntentResult
649
650 return stepResult
651
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700652def singleToMultiIntent( main,
653 name,
654 hostNames,
655 onosNode=0,
656 devices="",
657 ports=None,
658 ethType="",
659 macs=None,
660 bandwidth="",
661 lambdaAlloc=False,
662 ipProto="",
663 ipAddresses="",
664 tcp="",
665 sw1="",
666 sw2="",
667 expectedLink=0 ):
668 """
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700669 Verify Single to Multi Point intents
670 NOTE:If main.hostsData is not defined, variables data should be passed
671 in the same order index wise. All devices in the list should have the same
672 format, either all the devices have its port or it doesn't.
673 eg. hostName = [ 'h1', 'h2' ,.. ]
674 devices = [ 'of:0000000000000001', 'of:0000000000000002', ...]
675 ports = [ '1', '1', ..]
676 ...
677 Description:
678 Verify add-single-to-multi-intent iterates through the list of given
679 host | devices and add intents
680 Steps:
681 - Get device ids | ports
682 - Add single to multi point intents
683 - Check intents
684 - Verify flows
685 - Ping hosts
686 - Reroute
687 - Link down
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700688 - Verify flows
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700689 - Check topology
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700690 - Ping hosts
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700691 - Link up
692 - Verify flows
693 - Check topology
694 - Ping hosts
695 - Remove intents
696 Required:
697 name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
698 hostNames - List of host names
699 Optional:
700 onosNode - ONOS node to install the intents in main.CLIs[ ]
701 0 by default so that it will always use the first
702 ONOS node
703 devices - List of device ids in the same order as the hosts
704 in hostNames
705 ports - List of port numbers in the same order as the device in
706 devices
707 ethType - Ethernet type eg. IPV4, IPV6
708 macs - List of hosts mac address in the same order as the hosts in
709 hostNames
710 bandwidth - Bandwidth capacity
711 lambdaAlloc - Allocate lambda, defaults to False
712 ipProto - IP protocol
713 ipAddresses - IP addresses of host in the same order as the hosts in
714 hostNames
715 tcp - TCP ports in the same order as the hosts in hostNames
716 sw1 - First switch to bring down & up for rerouting purpose
717 sw2 - Second switch to bring down & up for rerouting purpose
718 expectedLink - Expected link when the switches are down, it should
719 be two links lower than the links before the two
720 switches are down
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700721 """
722
723 assert main, "There is no main variable"
724 assert hostNames, "You must specify hosts"
725 assert devices or main.hostsData, "You must specify devices"
726
727 global itemName
728 itemName = name
729 tempHostsData = {}
730 intentsId = []
731 onosNode = int( onosNode )
732
733 macsDict = {}
734 ipDict = {}
735 if hostNames and devices:
736 if len( hostNames ) != len( devices ):
737 main.log.debug( "hosts and devices does not have the same length" )
738 #print "len hostNames = ", len( hostNames )
739 #print "len devices = ", len( devices )
740 return main.FALSE
741 if ports:
742 if len( ports ) != len( devices ):
743 main.log.error( "Ports and devices does " +
744 "not have the same length" )
745 #print "len devices = ", len( devices )
746 #print "len ports = ", len( ports )
747 return main.FALSE
748 else:
749 main.log.info( "Device Ports are not specified" )
750 if macs:
751 for i in range( len( devices ) ):
752 macsDict[ devices[ i ] ] = macs[ i ]
753
754 elif hostNames and not devices and main.hostsData:
755 devices = []
756 main.log.info( "singleToMultiIntent function is using main.hostsData" )
757 for host in hostNames:
758 devices.append( main.hostsData.get( host ).get( 'location' ) )
759 macsDict[ main.hostsData.get( host ).get( 'location' ) ] = \
760 main.hostsData.get( host ).get( 'mac' )
761 ipDict[ main.hostsData.get( host ).get( 'location' ) ] = \
762 main.hostsData.get( host ).get( 'ipAddresses' )
763 #print main.hostsData
764
765 #print 'host names = ', hostNames
766 #print 'devices = ', devices
767 #print "macsDict = ", macsDict
768
769 pingResult = main.TRUE
770 intentResult = main.TRUE
771 removeIntentResult = main.TRUE
772 flowResult = main.TRUE
773 topoResult = main.TRUE
774 linkDownResult = main.TRUE
775 linkUpResult = main.TRUE
776
777 devicesCopy = copy.copy( devices )
778 if ports:
779 portsCopy = copy.copy( ports )
780 main.log.info( itemName + ": Adding single point to multi point intents" )
781
782 # Check flows count in each node
783 checkFlowsCount( main )
784
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700785 # Adding bidirectional point intents
786 for i in range( len( devices ) ):
787 ingressDevice = devicesCopy[ i ]
788 egressDeviceList = copy.copy( devicesCopy )
789 egressDeviceList.remove( ingressDevice )
790 if ports:
791 portIngress = portsCopy[ i ]
792 portEgressList = copy.copy( portsCopy )
793 del portEgressList[ i ]
794 else:
795 portIngress = ""
796 portEgressList = None
797 if not macsDict:
798 srcMac = ""
799 else:
800 srcMac = macsDict[ ingressDevice ]
801 if srcMac == None:
802 main.log.debug( "There is no MAC in device - " + ingressDevice )
803 srcMac = ""
804
805 intentsId.append(
806 main.CLIs[ onosNode ].addSinglepointToMultipointIntent(
807 ingressDevice=ingressDevice,
808 egressDeviceList=egressDeviceList,
809 portIngress=portIngress,
810 portEgressList=portEgressList,
811 ethType=ethType,
812 ethSrc=srcMac,
813 bandwidth=bandwidth,
814 lambdaAlloc=lambdaAlloc,
815 ipProto=ipProto,
816 ipSrc="",
817 ipDst="",
818 tcpSrc="",
819 tcpDst="" ) )
820
821 # Wait some time for the flow to go through when using multi instance
822 pingResult = pingallHosts( main, hostNames )
823
824 # Check intents state
825 time.sleep( main.checkIntentSleep )
826 intentResult = checkIntentState( main, intentsId )
827
828 # Check intents state again if first check fails...
829 if not intentResult:
830 intentResult = checkIntentState( main, intentsId )
831
832 # Check flows count in each node
833 checkFlowsCount( main )
834 # Verify flows
835 checkFlowsState( main )
836
837 pingResult = pingResult and pingallHosts( main, hostNames )
838
839 # Test rerouting if these variables exist
840 if sw1 and sw2 and expectedLink:
841 # link down
842 linkDownResult = link( main, sw1, sw2, "down" )
843 intentResult = intentResult and checkIntentState( main, intentsId )
844
845 # Check flows count in each node
846 checkFlowsCount( main )
847 # Verify flows
848 checkFlowsState( main )
849
850 # Check OnosTopology
851 topoResult = checkTopology( main, expectedLink )
852
853 # Ping hosts
854 pingResult = pingResult and pingallHosts( main, hostNames )
855
856 intentResult = checkIntentState( main, intentsId )
857
858 # Checks ONOS state in link down
859 if linkDownResult and topoResult and pingResult and intentResult:
860 main.log.info( itemName + ": Successfully brought link down" )
861 else:
862 main.log.error( itemName + ": Failed to bring link down" )
863
864 # link up
865 linkUpResult = link( main, sw1, sw2, "up" )
866 time.sleep( main.rerouteSleep )
867
868 # Check flows count in each node
869 checkFlowsCount( main )
870 # Verify flows
871 checkFlowsState( main )
872
873 # Check OnosTopology
874 topoResult = checkTopology( main, main.numLinks )
875
876 # Ping hosts
877 pingResult = pingResult and pingallHosts( main, hostNames )
878
879 intentResult = checkIntentState( main, intentsId )
880
881 # Checks ONOS state in link up
882 if linkUpResult and topoResult and pingResult and intentResult:
883 main.log.info( itemName + ": Successfully brought link back up" )
884 else:
885 main.log.error( itemName + ": Failed to bring link back up" )
886
887 # Remove all intents
888 removeIntentResult = removeAllIntents( main, intentsId )
889
890 stepResult = pingResult and linkDownResult and linkUpResult \
891 and intentResult and removeIntentResult
892
893 return stepResult
894
895def multiToSingleIntent( main,
896 name,
897 hostNames,
898 onosNode=0,
899 devices="",
900 ports=None,
901 ethType="",
902 macs=None,
903 bandwidth="",
904 lambdaAlloc=False,
905 ipProto="",
906 ipAddresses="",
907 tcp="",
908 sw1="",
909 sw2="",
910 expectedLink=0 ):
911 """
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700912 Verify Single to Multi Point intents
913 NOTE:If main.hostsData is not defined, variables data should be passed in the
914 same order index wise. All devices in the list should have the same
915 format, either all the devices have its port or it doesn't.
916 eg. hostName = [ 'h1', 'h2' ,.. ]
917 devices = [ 'of:0000000000000001', 'of:0000000000000002', ...]
918 ports = [ '1', '1', ..]
919 ...
920 Description:
921 Verify add-multi-to-single-intent
922 Steps:
923 - Get device ids | ports
924 - Add multi to single point intents
925 - Check intents
926 - Verify flows
927 - Ping hosts
928 - Reroute
929 - Link down
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700930 - Verify flows
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700931 - Check topology
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700932 - Ping hosts
kelvin-onlab58dc39e2015-08-06 08:11:09 -0700933 - Link up
934 - Verify flows
935 - Check topology
936 - Ping hosts
937 - Remove intents
938 Required:
939 name - Type of point intent to add eg. IPV4 | VLAN | Dualstack
940 hostNames - List of host names
941 Optional:
942 onosNode - ONOS node to install the intents in main.CLIs[ ]
943 0 by default so that it will always use the first
944 ONOS node
945 devices - List of device ids in the same order as the hosts
946 in hostNames
947 ports - List of port numbers in the same order as the device in
948 devices
949 ethType - Ethernet type eg. IPV4, IPV6
950 macs - List of hosts mac address in the same order as the hosts in
951 hostNames
952 bandwidth - Bandwidth capacity
953 lambdaAlloc - Allocate lambda, defaults to False
954 ipProto - IP protocol
955 ipAddresses - IP addresses of host in the same order as the hosts in
956 hostNames
957 tcp - TCP ports in the same order as the hosts in hostNames
958 sw1 - First switch to bring down & up for rerouting purpose
959 sw2 - Second switch to bring down & up for rerouting purpose
960 expectedLink - Expected link when the switches are down, it should
961 be two links lower than the links before the two
962 switches are down
kelvin-onlabd48a68c2015-07-13 16:01:36 -0700963 """
964
965 assert main, "There is no main variable"
966 assert hostNames, "You must specify hosts"
967 assert devices or main.hostsData, "You must specify devices"
968
969 global itemName
970 itemName = name
971 tempHostsData = {}
972 intentsId = []
973 onosNode = int( onosNode )
974
975 macsDict = {}
976 ipDict = {}
977 if hostNames and devices:
978 if len( hostNames ) != len( devices ):
979 main.log.debug( "hosts and devices does not have the same length" )
980 #print "len hostNames = ", len( hostNames )
981 #print "len devices = ", len( devices )
982 return main.FALSE
983 if ports:
984 if len( ports ) != len( devices ):
985 main.log.error( "Ports and devices does " +
986 "not have the same length" )
987 #print "len devices = ", len( devices )
988 #print "len ports = ", len( ports )
989 return main.FALSE
990 else:
991 main.log.info( "Device Ports are not specified" )
992 if macs:
993 for i in range( len( devices ) ):
994 macsDict[ devices[ i ] ] = macs[ i ]
995 elif hostNames and not devices and main.hostsData:
996 devices = []
997 main.log.info( "multiToSingleIntent function is using main.hostsData" )
998 for host in hostNames:
999 devices.append( main.hostsData.get( host ).get( 'location' ) )
1000 macsDict[ main.hostsData.get( host ).get( 'location' ) ] = \
1001 main.hostsData.get( host ).get( 'mac' )
1002 ipDict[ main.hostsData.get( host ).get( 'location' ) ] = \
1003 main.hostsData.get( host ).get( 'ipAddresses' )
1004 #print main.hostsData
1005
1006 #print 'host names = ', hostNames
1007 #print 'devices = ', devices
1008 #print "macsDict = ", macsDict
1009
1010 pingResult = main.TRUE
1011 intentResult = main.TRUE
1012 removeIntentResult = main.TRUE
1013 flowResult = main.TRUE
1014 topoResult = main.TRUE
1015 linkDownResult = main.TRUE
1016 linkUpResult = main.TRUE
1017
1018 devicesCopy = copy.copy( devices )
1019 if ports:
1020 portsCopy = copy.copy( ports )
1021 main.log.info( itemName + ": Adding multi point to single point intents" )
1022
1023 # Check flows count in each node
1024 checkFlowsCount( main )
1025
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001026 # Adding bidirectional point intents
1027 for i in range( len( devices ) ):
1028 egressDevice = devicesCopy[ i ]
1029 ingressDeviceList = copy.copy( devicesCopy )
1030 ingressDeviceList.remove( egressDevice )
1031 if ports:
1032 portEgress = portsCopy[ i ]
1033 portIngressList = copy.copy( portsCopy )
1034 del portIngressList[ i ]
1035 else:
1036 portEgress = ""
1037 portIngressList = None
1038 if not macsDict:
1039 dstMac = ""
1040 else:
1041 dstMac = macsDict[ egressDevice ]
1042 if dstMac == None:
1043 main.log.debug( "There is no MAC in device - " + egressDevice )
1044 dstMac = ""
1045
1046 intentsId.append(
1047 main.CLIs[ onosNode ].addMultipointToSinglepointIntent(
1048 ingressDeviceList=ingressDeviceList,
1049 egressDevice=egressDevice,
1050 portIngressList=portIngressList,
1051 portEgress=portEgress,
1052 ethType=ethType,
1053 ethDst=dstMac,
1054 bandwidth=bandwidth,
1055 lambdaAlloc=lambdaAlloc,
1056 ipProto=ipProto,
1057 ipSrc="",
1058 ipDst="",
1059 tcpSrc="",
1060 tcpDst="" ) )
1061
1062 pingResult = pingallHosts( main, hostNames )
1063
1064 # Check intents state
1065 time.sleep( main.checkIntentSleep )
1066 intentResult = checkIntentState( main, intentsId )
1067
1068 # Check intents state again if first check fails...
1069 if not intentResult:
1070 intentResult = checkIntentState( main, intentsId )
1071
1072 # Check flows count in each node
1073 checkFlowsCount( main )
1074 # Verify flows
1075 checkFlowsState( main )
1076
1077 # Ping hosts
1078 pingResult = pingResult and pingallHosts( main, hostNames )
1079 # Ping hosts again...
1080 pingResult = pingResult and pingallHosts( main, hostNames )
1081
1082 # Test rerouting if these variables exist
1083 if sw1 and sw2 and expectedLink:
1084 # link down
1085 linkDownResult = link( main, sw1, sw2, "down" )
1086 intentResult = intentResult and checkIntentState( main, intentsId )
1087
1088 # Check flows count in each node
1089 checkFlowsCount( main )
1090 # Verify flows
1091 checkFlowsState( main )
1092
1093 # Check OnosTopology
1094 topoResult = checkTopology( main, expectedLink )
1095
1096 # Ping hosts
1097 pingResult = pingResult and pingallHosts( main, hostNames )
1098
1099 intentResult = checkIntentState( main, intentsId )
1100
1101 # Checks ONOS state in link down
1102 if linkDownResult and topoResult and pingResult and intentResult:
1103 main.log.info( itemName + ": Successfully brought link down" )
1104 else:
1105 main.log.error( itemName + ": Failed to bring link down" )
1106
1107 # link up
1108 linkUpResult = link( main, sw1, sw2, "up" )
1109 time.sleep( main.rerouteSleep )
1110
1111 # Check flows count in each node
1112 checkFlowsCount( main )
1113 # Verify flows
1114 checkFlowsState( main )
1115
1116 # Check OnosTopology
1117 topoResult = checkTopology( main, main.numLinks )
1118
1119 # Ping hosts
1120 pingResult = pingResult and pingallHosts( main, hostNames )
1121
1122 intentResult = checkIntentState( main, intentsId )
1123
1124 # Checks ONOS state in link up
1125 if linkUpResult and topoResult and pingResult and intentResult:
1126 main.log.info( itemName + ": Successfully brought link back up" )
1127 else:
1128 main.log.error( itemName + ": Failed to bring link back up" )
1129
1130 # Remove all intents
1131 removeIntentResult = removeAllIntents( main, intentsId )
1132
1133 stepResult = pingResult and linkDownResult and linkUpResult \
1134 and intentResult and removeIntentResult
1135
1136 return stepResult
1137
kelvin-onlab58dc39e2015-08-06 08:11:09 -07001138def pingallHosts( main, hostList ):
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001139 # Ping all host in the hosts list variable
1140 print "Pinging : ", hostList
1141 pingResult = main.TRUE
kelvin-onlab58dc39e2015-08-06 08:11:09 -07001142 pingResult = main.Mininet1.pingallHosts( hostList )
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001143 return pingResult
1144
1145def getHostsData( main ):
1146 """
1147 Use fwd app and pingall to discover all the hosts
1148 """
1149
1150 activateResult = main.TRUE
1151 appCheck = main.TRUE
1152 getDataResult = main.TRUE
1153 main.log.info( "Activating reactive forwarding app " )
1154 activateResult = main.CLIs[ 0 ].activateApp( "org.onosproject.fwd" )
kelvin-onlab0ad05d12015-07-23 14:21:15 -07001155 time.sleep( main.fwdSleep )
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001156
1157 for i in range( main.numCtrls ):
1158 appCheck = appCheck and main.CLIs[ i ].appToIDCheck()
1159 if appCheck != main.TRUE:
1160 main.log.warn( main.CLIs[ i ].apps() )
1161 main.log.warn( main.CLIs[ i ].appIDs() )
1162
1163 pingResult = main.Mininet1.pingall( timeout = 600 )
1164 hostsJson = json.loads( main.CLIs[ 0 ].hosts() )
1165 hosts = main.Mininet1.getHosts().keys()
1166 # TODO: Make better use of new getHosts function
1167 for host in hosts:
1168 main.hostsData[ host ] = {}
1169 main.hostsData[ host ][ 'mac' ] = \
1170 main.Mininet1.getMacAddress( host ).upper()
1171 for hostj in hostsJson:
1172 if main.hostsData[ host ][ 'mac' ] == hostj[ 'mac' ]:
1173 main.hostsData[ host ][ 'id' ] = hostj[ 'id' ]
1174 main.hostsData[ host ][ 'vlan' ] = hostj[ 'vlan' ]
1175 main.hostsData[ host ][ 'location' ] = \
1176 hostj[ 'location' ][ 'elementId' ] + '/' + \
1177 hostj[ 'location' ][ 'port' ]
1178 main.hostsData[ host ][ 'ipAddresses' ] = hostj[ 'ipAddresses' ]
1179
1180 main.log.info( "Deactivating reactive forwarding app " )
1181 deactivateResult = main.CLIs[ 0 ].deactivateApp( "org.onosproject.fwd" )
1182 if activateResult and deactivateResult and main.hostsData:
1183 main.log.info( "Successfully used fwd app to discover hosts " )
1184 getDataResult = main.TRUE
1185 else:
1186 main.log.info( "Failed to use fwd app to discover hosts " )
1187 getDataResult = main.FALSE
1188
1189 print main.hostsData
1190
1191 return getDataResult
1192
1193def checkTopology( main, expectedLink ):
1194 statusResult = main.TRUE
1195 # Check onos topology
1196 main.log.info( itemName + ": Checking ONOS topology " )
1197
1198 for i in range( main.numCtrls ):
1199 topologyResult = main.CLIs[ i ].topology()
1200 statusResult = main.ONOSbench.checkStatus( topologyResult,
1201 main.numSwitch,
1202 expectedLink )\
1203 and statusResult
1204 if not statusResult:
1205 main.log.error( itemName + ": Topology mismatch" )
1206 else:
1207 main.log.info( itemName + ": Topology match" )
1208 return statusResult
1209
1210def checkIntentState( main, intentsId ):
1211 """
1212 This function will check intent state to make sure all the intents
1213 are in INSTALLED state
1214 """
1215
1216 intentResult = main.TRUE
1217 results = []
1218
1219 main.log.info( itemName + ": Checking intents state" )
1220 # First check of intents
1221 for i in range( main.numCtrls ):
1222 tempResult = main.CLIs[ i ].checkIntentState( intentsId=intentsId )
1223 results.append( tempResult )
1224
1225 expectedState = [ 'INSTALLED', 'INSTALLING' ]
1226
1227 if all( result == main.TRUE for result in results ):
1228 main.log.info( itemName + ": Intents are installed correctly" )
1229 else:
1230 # Wait for at least 5 second before checking the intents again
1231 time.sleep( 5 )
1232 results = []
1233 # Second check of intents since some of the intents may be in
1234 # INSTALLING state, they should be in INSTALLED at this time
1235 for i in range( main.numCtrls ):
1236 tempResult = main.CLIs[ i ].checkIntentState(
1237 intentsId=intentsId )
1238 results.append( tempResult )
1239 if all( result == main.TRUE for result in results ):
1240 main.log.info( itemName + ": Intents are installed correctly" )
1241 else:
1242 main.log.error( itemName + ": Intents are NOT installed correctly" )
1243 intentResult = main.FALSE
1244
1245 return intentResult
1246
1247def checkFlowsState( main ):
1248
1249 main.log.info( itemName + ": Check flows state" )
1250 checkFlowsResult = main.CLIs[ 0 ].checkFlowsState()
1251 return checkFlowsResult
1252
1253def link( main, sw1, sw2, option):
1254
1255 # link down
1256 main.log.info( itemName + ": Bring link " + option + "between " +
1257 sw1 + " and " + sw2 )
1258 linkResult = main.Mininet1.link( end1=sw1, end2=sw2, option=option )
1259 return linkResult
1260
1261def removeAllIntents( main, intentsId ):
1262 """
1263 Remove all intents in the intentsId
1264 """
1265
kelvin-onlab5c706ff2015-07-20 10:56:52 -07001266 onosSummary = []
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001267 removeIntentResult = main.TRUE
1268 # Remove intents
1269 for intent in intentsId:
1270 main.CLIs[ 0 ].removeIntent( intentId=intent, purge=True )
1271
1272 time.sleep( 5 )
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001273
kelvin-onlab5c706ff2015-07-20 10:56:52 -07001274 # If there is remianing intents then remove intents should fail
1275 for i in range( main.numCtrls ):
1276 onosSummary.append( json.loads( main.CLIs[ i ].summary() ) )
1277
1278 for summary in onosSummary:
1279 if summary.get( 'intents' ) != 0:
1280 main.log.warn( itemName + ": There are " +
1281 str( summary.get( 'intents' ) ) +
1282 " intents remaining in node " +
1283 str( summary.get( 'node' ) ) +
1284 ", failed to remove all the intents " )
1285 removeIntentResult = main.FALSE
1286
1287 if removeIntentResult:
1288 main.log.info( itemName + ": There are no more intents remaining, " +
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001289 "successfully removed all the intents." )
kelvin-onlab5c706ff2015-07-20 10:56:52 -07001290
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001291 return removeIntentResult
1292
1293def checkFlowsCount( main ):
1294 """
1295 Check flows count in each node
1296 """
1297
1298 flowsCount = []
1299 main.log.info( itemName + ": Checking flows count in each ONOS node" )
1300 for i in range( main.numCtrls ):
1301 summaryResult = main.CLIs[ i ].summary()
1302 if not summaryResult:
1303 main.log.error( itemName + ": There is something wrong with " +
1304 "summary command" )
1305 return main.FALSE
1306 else:
1307 summaryJson = json.loads( summaryResult )
1308 flowsCount.append( summaryJson.get( 'flows' ) )
1309
1310 if flowsCount:
1311 if all( flows==flowsCount[ 0 ] for flows in flowsCount ):
1312 main.log.info( itemName + ": There are " + str( flowsCount[ 0 ] ) +
1313 " flows in all ONOS node" )
1314 else:
1315 for i in range( main.numCtrls ):
1316 main.log.debug( itemName + ": ONOS node " + str( i ) + " has " +
kelvin-onlab6dea6e62015-07-23 13:07:26 -07001317 str( flowsCount[ i ] ) + " flows" )
kelvin-onlabd48a68c2015-07-13 16:01:36 -07001318 else:
1319 main.log.error( "Checking flows count failed, check summary command" )
1320 return main.FALSE
1321
1322 return main.TRUE
1323
kelvin-onlab58dc39e2015-08-06 08:11:09 -07001324def checkLeaderChange( leaders1, leaders2 ):
acsmarse6b410f2015-07-17 14:39:34 -07001325 """
1326 Checks for a change in intent partition leadership.
1327
1328 Takes the output of leaders -c in json string format before and after
1329 a potential change as input
1330
1331 Returns main.TRUE if no mismatches are detected
1332 Returns main.FALSE if there is a mismatch or on error loading the input
1333 """
1334 try:
1335 leaders1 = json.loads( leaders1 )
1336 leaders2 = json.loads( leaders2 )
1337 except ( AttributeError, TypeError):
1338 main.log.exception( self.name + ": Object not as expected" )
1339 return main.FALSE
1340 except Exception:
1341 main.log.exception( self.name + ": Uncaught exception!" )
1342 main.cleanup()
1343 main.exit()
1344 main.log.info( "Checking Intent Paritions for Change in Leadership" )
1345 mismatch = False
1346 for dict1 in leaders1:
1347 if "intent" in dict1.get( "topic", [] ):
1348 for dict2 in leaders2:
1349 if dict1.get( "topic", 0 ) == dict2.get( "topic", 0 ) and \
1350 dict1.get( "leader", 0 ) != dict2.get( "leader", 0 ):
1351 mismatch = True
1352 main.log.error( "{0} changed leader from {1} to {2}".\
1353 format( dict1.get( "topic", "no-topic" ),\
1354 dict1.get( "leader", "no-leader" ),\
1355 dict2.get( "leader", "no-leader" ) ) )
1356 if mismatch:
1357 return main.FALSE
1358 else:
1359 return main.TRUE