blob: 5da5834f5b7342c30f176fd74fb748c17988bd96 [file] [log] [blame]
Jon Hall1efcb3f2016-08-23 13:42:15 -07001import os
2import imp
3import time
4import json
5import urllib
6from core import utilities
7
8
9class Testcaselib:
Pierfb719b12016-09-19 14:51:44 -070010
11 useSSH=False
12
Jon Hall1efcb3f2016-08-23 13:42:15 -070013 @staticmethod
14 def initTest( main ):
15 """
16 - Construct tests variables
17 - GIT ( optional )
18 - Checkout ONOS master branch
19 - Pull latest ONOS code
20 - Building ONOS ( optional )
21 - Install ONOS package
22 - Build ONOS package
23 """
24 main.step( "Constructing test variables" )
25 # Test variables
26 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
27 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
28 main.diff = main.params[ 'ENV' ][ 'diffApps' ]
29 gitBranch = main.params[ 'GIT' ][ 'branch' ]
30 main.path = os.path.dirname( main.testFile )
31 main.dependencyPath = main.path + "/../dependencies/"
32 main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
33 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
34 main.scale = (main.params[ 'SCALE' ][ 'size' ]).split( "," )
35 main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
36 # main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
37 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
38 main.cellData = { } # for creating cell file
39 main.CLIs = [ ]
40 main.ONOSip = [ ]
41 main.RESTs = [ ]
42
43 # Assigning ONOS cli handles to a list
44 for i in range( 1, main.maxNodes + 1 ):
45 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
46 main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) )
47 main.ONOSip.append( main.CLIs[ i - 1 ].ip_address )
48 # -- INIT SECTION, ONLY RUNS ONCE -- #
49 main.startUp = imp.load_source( wrapperFile1,
50 main.dependencyPath +
51 wrapperFile1 +
52 ".py" )
53
54 copyResult1 = main.ONOSbench.scp( main.Mininet1,
55 main.dependencyPath +
56 main.topology,
57 main.Mininet1.home,
58 direction="to" )
59 if main.CLIs:
60 stepResult = main.TRUE
61 else:
62 main.log.error( "Did not properly created list of ONOS CLI handle" )
63 stepResult = main.FALSE
64
65 utilities.assert_equals( expect=main.TRUE,
66 actual=stepResult,
67 onpass="Successfully construct " +
68 "test variables ",
69 onfail="Failed to construct test variables" )
70
71 @staticmethod
72 def installOnos( main, vlanCfg=True ):
73 """
74 - Set up cell
75 - Create cell file
76 - Set cell file
77 - Verify cell file
78 - Kill ONOS process
79 - Uninstall ONOS cluster
80 - Verify ONOS start up
81 - Install ONOS cluster
82 - Connect to cli
83 """
84 # main.scale[ 0 ] determines the current number of ONOS controller
85 apps = main.apps
86 if main.diff:
87 apps = main.apps + "," + main.diff
88 else:
89 main.log.error( "App list is empty" )
90 print "NODE COUNT = ", main.numCtrls
91 print main.ONOSip
92 tempOnosIp = [ ]
93 main.dynamicHosts = [ 'in1', 'out1' ]
94 for i in range( main.numCtrls ):
95 tempOnosIp.append( main.ONOSip[ i ] )
96 onosUser = main.params[ 'ENV' ][ 'cellUser' ]
97 main.step( "Create and Apply cell file" )
98 main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
99 "temp",
100 main.Mininet1.ip_address,
101 apps,
102 tempOnosIp,
103 onosUser,
Pierfb719b12016-09-19 14:51:44 -0700104 useSSH=Testcaselib.useSSH )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700105 cellResult = main.ONOSbench.setCell( "temp" )
106 verifyResult = main.ONOSbench.verifyCell( )
107 stepResult = cellResult and verifyResult
108 utilities.assert_equals( expect=main.TRUE,
109 actual=stepResult,
110 onpass="Successfully applied cell to " + \
111 "environment",
112 onfail="Failed to apply cell to environment " )
113 # kill off all onos processes
114 main.log.info( "Safety check, killing all ONOS processes" +
115 " before initiating environment setup" )
116 for i in range( main.maxNodes ):
117 main.ONOSbench.onosDie( main.ONOSip[ i ] )
118 main.step( "Create and Install ONOS package" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700119 packageResult = main.ONOSbench.buckBuild()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700120
121 onosInstallResult = main.TRUE
122 for i in range( main.numCtrls ):
123 onosInstallResult = onosInstallResult and \
124 main.ONOSbench.onosInstall(
125 node=main.ONOSip[ i ] )
126 stepResult = onosInstallResult
127 utilities.assert_equals( expect=main.TRUE,
128 actual=stepResult,
129 onpass="Successfully installed ONOS package",
130 onfail="Failed to install ONOS package" )
Pierfb719b12016-09-19 14:51:44 -0700131 if Testcaselib.useSSH:
132 for i in range( main.numCtrls ):
133 onosInstallResult = onosInstallResult and \
134 main.ONOSbench.onosSecureSSH(
135 node=main.ONOSip[ i ] )
136 stepResult = onosInstallResult
137 utilities.assert_equals( expect=main.TRUE,
138 actual=stepResult,
139 onpass="Successfully secure SSH",
140 onfail="Failed to secure SSH" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700141 main.step( "Starting ONOS service" )
142 stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE,
143 for i in range( main.numCtrls ):
144 onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
145 if onosIsUp == main.TRUE:
146 main.log.report( "ONOS instance is up and ready" )
147 else:
148 main.log.report( "ONOS instance may not be up, stop and " +
149 "start ONOS again " )
150 for i in range( main.numCtrls ):
151 stopResult = stopResult and \
152 main.ONOSbench.onosStop( main.ONOSip[ i ] )
153 for i in range( main.numCtrls ):
154 startResult = startResult and \
155 main.ONOSbench.onosStart( main.ONOSip[ i ] )
156 stepResult = onosIsUp and stopResult and startResult
157
158 utilities.assert_equals( expect=main.TRUE,
159 actual=stepResult,
160 onpass="ONOS service is ready",
161 onfail="ONOS service did not start properly" )
162 main.step( "Checking if ONOS CLI is ready" )
163 for i in range( main.numCtrls ):
164 main.CLIs[ i ].startCellCli( )
165 cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
166 commandlineTimeout=60,
167 onosStartTimeout=100 )
168 utilities.assert_equals( expect=main.TRUE,
169 actual=cliResult,
170 onpass="ONOS CLI is ready",
171 onfail="ONOS CLI is not ready" )
172 main.active = 0
173 for i in range( 10 ):
174 ready = True
175 output = main.CLIs[ main.active ].summary( )
176 if not output:
177 ready = False
178 if ready:
179 break
180 time.sleep( 10 )
181 utilities.assert_equals( expect=True, actual=ready,
182 onpass="ONOS summary command succeded",
183 onfail="ONOS summary command failed" )
184
185 with open( "%s/json/%s.json" % (
186 main.dependencyPath, main.cfgName) ) as cfg:
187 main.RESTs[ main.active ].setNetCfg( json.load( cfg ) )
188 with open( "%s/json/%s.chart" % (
189 main.dependencyPath, main.cfgName) ) as chart:
190 main.pingChart = json.load( chart )
191 if not ready:
192 main.log.error( "ONOS startup failed!" )
193 main.cleanup( )
194 main.exit( )
195
196 for i in range( main.numCtrls ):
197 main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.segmentrouting" )
198 main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.driver.pipeline" )
199 main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.store.group.impl" )
200 main.CLIs[ i ].logSet( "DEBUG",
201 "org.onosproject.net.flowobjective.impl" )
202
203 @staticmethod
204 def startMininet( main, topology, args="" ):
205 main.step( "Starting Mininet Topology" )
206 arg = "--onos %d %s" % (main.numCtrls, args)
207 main.topology = topology
208 topoResult = main.Mininet1.startNet(
209 topoFile=main.Mininet1.home + main.topology, args=arg )
210 stepResult = topoResult
211 utilities.assert_equals( expect=main.TRUE,
212 actual=stepResult,
213 onpass="Successfully loaded topology",
214 onfail="Failed to load topology" )
215 # Exit if topology did not load properly
216 if not topoResult:
217 main.cleanup( )
218 main.exit( )
219
220 @staticmethod
221 def checkFlows( main, minFlowCount, dumpflows=True ):
222 main.step(
223 " Check whether the flow count is bigger than %s" % minFlowCount )
224 count = utilities.retry( main.CLIs[ main.active ].checkFlowCount,
225 main.FALSE,
226 kwargs={ 'min': minFlowCount },
227 attempts=10,
228 sleep=10 )
229 utilities.assertEquals( \
230 expect=True,
231 actual=(count > 0),
232 onpass="Flow count looks correct: " + str( count ),
233 onfail="Flow count looks wrong: " + str( count ) )
234
235 main.step( "Check whether all flow status are ADDED" )
236 flowCheck = utilities.retry( main.CLIs[ main.active ].checkFlowsState,
237 main.FALSE,
238 kwargs={ 'isPENDING': False },
239 attempts=2,
240 sleep=10 )
241 utilities.assertEquals( \
242 expect=main.TRUE,
243 actual=flowCheck,
244 onpass="Flow status is correct!",
245 onfail="Flow status is wrong!" )
246 if dumpflows:
Pier50f0bc62016-09-07 17:53:40 -0700247 main.ONOSbench.dumpONOSCmd( main.ONOSip[ main.active ],
248 "flows",
249 main.logdir,
250 "flowsBefore" + main.cfgName )
251 main.ONOSbench.dumpONOSCmd( main.ONOSip[ main.active ],
252 "groups",
253 main.logdir,
254 "groupsBefore" + main.cfgName )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700255
256 @staticmethod
257 def pingAll( main, tag="", dumpflows=True ):
258 main.log.report( "Check full connectivity" )
259 print main.pingChart
260 for entry in main.pingChart.itervalues( ):
261 print entry
262 hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
263 expect = main.TRUE if expect else main.FALSE
264 main.step( "Connectivity for %s %s" % (str( hosts ), tag) )
265 pa = main.Mininet1.pingallHosts( hosts )
266 utilities.assert_equals( expect=expect, actual=pa,
267 onpass="IP connectivity successfully tested",
268 onfail="IP connectivity failed" )
269 if dumpflows:
Pier50f0bc62016-09-07 17:53:40 -0700270 main.ONOSbench.dumpONOSCmd( main.ONOSip[ main.active ],
271 "flows",
272 main.logdir,
273 "flowsOn" + tag )
274 main.ONOSbench.dumpONOSCmd( main.ONOSip[ main.active ],
275 "groups",
276 main.logdir,
277 "groupsOn" + tag )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700278
279 @staticmethod
280 def killLink( main, end1, end2, switches, links ):
281 """
282 end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
283 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
284 Kill a link and verify ONOS can see the proper link change
285 """
286 main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
287 main.step( "Kill link between %s and %s" % (end1, end2) )
288 LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
Pierfb719b12016-09-19 14:51:44 -0700289 LinkDown = main.Mininet1.link( END2=end1, END1=end2, OPTION="down" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700290 main.log.info(
291 "Waiting %s seconds for link down to be discovered" % main.linkSleep )
292 time.sleep( main.linkSleep )
293 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
294 main.FALSE,
295 kwargs={ 'numoswitch': switches,
296 'numolink': links },
297 attempts=10,
298 sleep=main.linkSleep )
299 result = topology & LinkDown
300 utilities.assert_equals( expect=main.TRUE, actual=result,
301 onpass="Link down successful",
302 onfail="Failed to turn off link?" )
303
304 @staticmethod
305 def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
306 links ):
307 """
308 Params:
309 end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
310 dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
311 port1, port2: respective port of the end switches that connects to the link, ex.:'1'
312 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
313 Kill a link and verify ONOS can see the proper link change
314 """
315 main.step( "Restore link between %s and %s" % (end1, end2) )
316 result = False
317 count = 0
318 while True:
319 count += 1
320 main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
321 main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
322 main.log.info(
323 "Waiting %s seconds for link up to be discovered" % main.linkSleep )
324 time.sleep( main.linkSleep )
Pierfb719b12016-09-19 14:51:44 -0700325
326 for i in range(0, main.numCtrls):
327 onosIsUp = main.ONOSbench.isup( main.ONOSip[ i ] )
328 if onosIsUp == main.TRUE:
329 main.CLIs[ i ].portstate( dpid=dpid1, port=port1 )
330 main.CLIs[ i ].portstate( dpid=dpid2, port=port2 )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700331 time.sleep( main.linkSleep )
332
333 result = main.CLIs[ main.active ].checkStatus( numoswitch=switches,
334 numolink=links )
335 if count > 5 or result:
336 break
337 utilities.assert_equals( expect=main.TRUE, actual=result,
338 onpass="Link up successful",
339 onfail="Failed to bring link up" )
340
341 @staticmethod
342 def killSwitch( main, switch, switches, links ):
343 """
344 Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
345 Completely kill a switch and verify ONOS can see the proper change
346 """
347 main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
348 main.step( "Kill " + switch )
349 main.log.info( "Stopping" + switch )
350 main.Mininet1.switch( SW=switch, OPTION="stop" )
351 # todo make this repeatable
352 main.log.info( "Waiting %s seconds for switch down to be discovered" % (
353 main.switchSleep) )
354 time.sleep( main.switchSleep )
355 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
356 main.FALSE,
357 kwargs={ 'numoswitch': switches,
358 'numolink': links },
359 attempts=10,
360 sleep=main.switchSleep )
361 utilities.assert_equals( expect=main.TRUE, actual=topology,
362 onpass="Kill switch successful",
363 onfail="Failed to kill switch?" )
364
365 @staticmethod
366 def recoverSwitch( main, switch, switches, links ):
367 """
368 Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
369 Recover a switch and verify ONOS can see the proper change
370 """
371 # todo make this repeatable
372 main.step( "Recovering " + switch )
373 main.log.info( "Starting" + switch )
374 main.Mininet1.switch( SW=switch, OPTION="start" )
375 main.log.info( "Waiting %s seconds for switch up to be discovered" % (
376 main.switchSleep) )
377 time.sleep( main.switchSleep )
378 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
379 main.FALSE,
380 kwargs={ 'numoswitch': switches,
381 'numolink': links },
382 attempts=10,
383 sleep=main.switchSleep )
384 utilities.assert_equals( expect=main.TRUE, actual=topology,
385 onpass="Switch recovery successful",
386 onfail="Failed to recover switch?" )
387
388 @staticmethod
389 def cleanup( main ):
390 """
391 Stop Onos-cluster.
392 Stops Mininet
393 Copies ONOS log
394 """
395 main.Mininet1.stopNet( )
396 main.ONOSbench.scp( main.ONOScli1, "/opt/onos/log/karaf.log",
397 "/tmp/karaf.log", direction="from" )
398 main.ONOSbench.cpLogsToDir( "/tmp/karaf.log", main.logdir,
399 copyFileName="karaf.log." + main.cfgName )
400 for i in range( main.numCtrls ):
401 main.ONOSbench.onosStop( main.ONOSip[ i ] )
402
403 @staticmethod
404 def killOnos( main, nodes, switches, links, expNodes ):
405 """
406 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
407 switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
408 Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
409 """
410 main.step( "Killing ONOS instance" )
411 for i in nodes:
412 killResult = main.ONOSbench.onosDie( main.CLIs[ i ].ip_address )
413 utilities.assert_equals( expect=main.TRUE, actual=killResult,
414 onpass="ONOS instance Killed",
415 onfail="Error killing ONOS instance" )
416 if i == main.active:
417 main.active = (i + 1) % main.numCtrls
418 time.sleep( 12 )
419 if len( nodes ) < main.numCtrls:
420 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
421 main.FALSE,
422 kwargs={ 'numoswitch': switches,
423 'numolink': links,
424 'numoctrl': expNodes },
425 attempts=10,
426 sleep=12 )
427 utilities.assert_equals( expect=main.TRUE, actual=topology,
428 onpass="ONOS Instance down successful",
429 onfail="Failed to turn off ONOS Instance" )
430 else:
431 main.active = -1
432
433 @staticmethod
434 def recoverOnos( main, nodes, switches, links, expNodes ):
435 """
436 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
437 switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
438 Recover an ONOS instance and verify the ONOS cluster can see the proper change
439 """
440 main.step( "Recovering ONOS instance" )
441 [ main.ONOSbench.onosStart( main.CLIs[ i ].ip_address ) for i in nodes ]
442 for i in nodes:
443 isUp = main.ONOSbench.isup( main.ONOSip[ i ] )
444 utilities.assert_equals( expect=main.TRUE, actual=isUp,
445 onpass="ONOS service is ready",
446 onfail="ONOS service did not start properly" )
447 for i in nodes:
448 main.step( "Checking if ONOS CLI is ready" )
449 main.CLIs[ i ].startCellCli( )
450 cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
451 commandlineTimeout=60,
452 onosStartTimeout=100 )
453 utilities.assert_equals( expect=main.TRUE,
454 actual=cliResult,
455 onpass="ONOS CLI is ready",
456 onfail="ONOS CLI is not ready" )
457 main.active = i if main.active == -1 else main.active
458
459 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
460 main.FALSE,
461 kwargs={ 'numoswitch': switches,
462 'numolink': links,
463 'numoctrl': expNodes },
464 attempts=10,
465 sleep=12 )
466 utilities.assert_equals( expect=main.TRUE, actual=topology,
467 onpass="ONOS Instance down successful",
468 onfail="Failed to turn off ONOS Instance" )
469 for i in range( 10 ):
470 ready = True
471 output = main.CLIs[ main.active ].summary( )
472 if not output:
473 ready = False
474 if ready:
475 break
476 time.sleep( 10 )
477 utilities.assert_equals( expect=True, actual=ready,
478 onpass="ONOS summary command succeded",
479 onfail="ONOS summary command failed" )
480 if not ready:
481 main.log.error( "ONOS startup failed!" )
482 main.cleanup( )
483 main.exit( )
484
485 @staticmethod
486 def addHostCfg( main ):
487 """
488 Adds Host Configuration to ONOS
489 Updates expected state of the network (pingChart)
490 """
491 import json
492 hostCfg = { }
493 with open( main.dependencyPath + "/json/extra.json" ) as template:
494 hostCfg = json.load( template )
495 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'in1' ]
496 main.step( "Pushing new configuration" )
497 mac, cfg = hostCfg[ 'hosts' ].popitem( )
498 main.RESTs[ main.active ].setNetCfg( cfg[ 'basic' ],
499 subjectClass="hosts",
500 subjectKey=urllib.quote( mac,
501 safe='' ),
502 configKey="basic" )
503 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'out1' ]
504 main.step( "Pushing new configuration" )
505 mac, cfg = hostCfg[ 'hosts' ].popitem( )
506 main.RESTs[ main.active ].setNetCfg( cfg[ 'basic' ],
507 subjectClass="hosts",
508 subjectKey=urllib.quote( mac,
509 safe='' ),
510 configKey="basic" )
511 main.pingChart.update( { 'vlan1': { "expect": "True",
512 "hosts": [ "olt1", "vsg1" ] } } )
513 main.pingChart[ 'vlan5' ][ 'expect' ] = 0
514 main.pingChart[ 'vlan10' ][ 'expect' ] = 0
515 ports = "[%s,%s]" % (5, 6)
516 cfg = '{"of:0000000000000001":[{"vlan":1,"ports":%s,"name":"OLT 1"}]}' % ports
517 main.RESTs[ main.active ].setNetCfg( json.loads( cfg ),
518 subjectClass="apps",
519 subjectKey="org.onosproject.segmentrouting",
520 configKey="xconnect" )
521
522 @staticmethod
523 def delHostCfg( main ):
524 """
525 Removest Host Configuration from ONOS
526 Updates expected state of the network (pingChart)
527 """
528 import json
529 hostCfg = { }
530 with open( main.dependencyPath + "/json/extra.json" ) as template:
531 hostCfg = json.load( template )
532 main.step( "Removing host configuration" )
533 main.pingChart[ 'ip' ][ 'expect' ] = 0
534 mac, cfg = hostCfg[ 'hosts' ].popitem( )
535 main.RESTs[ main.active ].removeNetCfg( subjectClass="hosts",
536 subjectKey=urllib.quote(
537 mac,
538 safe='' ),
539 configKey="basic" )
540 main.step( "Removing configuration" )
541 main.pingChart[ 'ip' ][ 'expect' ] = 0
542 mac, cfg = hostCfg[ 'hosts' ].popitem( )
543 main.RESTs[ main.active ].removeNetCfg( subjectClass="hosts",
544 subjectKey=urllib.quote(
545 mac,
546 safe='' ),
547 configKey="basic" )
548 main.step( "Removing vlan configuration" )
549 main.pingChart[ 'vlan1' ][ 'expect' ] = 0
550 main.RESTs[ main.active ].removeNetCfg( subjectClass="apps",
551 subjectKey="org.onosproject.segmentrouting",
552 configKey="xconnect" )