blob: 8664f21896ab9b1fe79d4529b5fda3543812fda1 [file] [log] [blame]
kelvin-onlab1d381fe2015-07-14 16:24:56 -07001
2# Testing network scalability, this test suite scales up a network topology
3# using mininet and verifies ONOS stability
4
GlennRC1c5df3c2015-08-27 16:12:09 -07005class SCPFscaleTopo:
kelvin-onlab1d381fe2015-07-14 16:24:56 -07006
7 def __init__( self ):
8 self.default = ''
9
10 def CASE1( self, main ):
11 import time
12 import os
13 import imp
Jon Hallf632d202015-07-30 15:45:11 -070014 import re
kelvin-onlab1d381fe2015-07-14 16:24:56 -070015
16 """
17 - Construct tests variables
18 - GIT ( optional )
19 - Checkout ONOS master branch
20 - Pull latest ONOS code
21 - Building ONOS ( optional )
22 - Install ONOS package
23 - Build ONOS package
24 """
25
GlennRC475f50d2015-10-23 15:01:09 -070026 main.case( "Constructing test variables" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070027 main.step( "Constructing test variables" )
28 stepResult = main.FALSE
29
GlennRC475f50d2015-10-23 15:01:09 -070030 main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
31 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
32 gitBranch = main.params[ 'GIT' ][ 'branch' ]
33 main.dependencyPath = main.testOnDirectory + \
34 main.params[ 'DEPENDENCY' ][ 'path' ]
35 main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
36 main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
37 main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] )
38 main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
39 main.topoScaleSize = len( main.topoScale )
40 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
41 wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
42 wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
43 main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] )
44 main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] )
45 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
46 main.fwdSleep = int( main.params[ 'SLEEP' ][ 'fwd' ] )
47 main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
48 main.nodeDownSleep = int( main.params[ 'SLEEP' ][ 'nodeDown' ] )
49 main.nodeUpSleep = int( main.params[ 'SLEEP' ][ 'nodeUp' ] )
50 main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] )
51 main.stopMNSleep = int( main.params[ 'SLEEP' ][ 'stopMN' ] )
52 main.startMNSleep = int( main.params[ 'SLEEP' ][ 'startMN' ] )
53 main.pingTimeout = int( main.params[ 'TIMEOUT' ][ 'pingall' ] )
54 gitPull = main.params[ 'GIT' ][ 'pull' ]
55 main.homeDir = os.path.expanduser('~')
56 main.cellData = {} # for creating cell file
57 main.hostsData = {}
58 main.CLIs = []
59 main.ONOSip = []
60 main.activeNodes = []
61 main.ONOSip = main.ONOSbench.getOnosIps()
kelvin-onlab1d381fe2015-07-14 16:24:56 -070062
GlennRC475f50d2015-10-23 15:01:09 -070063 for i in range(main.numCtrls):
GlennRC632e2892015-10-19 18:58:41 -070064 main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
65
GlennRC475f50d2015-10-23 15:01:09 -070066 main.startUp = imp.load_source( wrapperFile1,
67 main.dependencyPath +
68 wrapperFile1 +
69 ".py" )
GlennRC632e2892015-10-19 18:58:41 -070070
GlennRC475f50d2015-10-23 15:01:09 -070071 main.scaleTopoFunction = imp.load_source( wrapperFile2,
72 main.dependencyPath +
73 wrapperFile2 +
74 ".py" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070075
GlennRC475f50d2015-10-23 15:01:09 -070076 main.topo = imp.load_source( wrapperFile3,
77 main.dependencyPath +
78 wrapperFile3 +
79 ".py" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070080
GlennRC632e2892015-10-19 18:58:41 -070081 main.ONOSbench.scp( main.Mininet1,
82 main.dependencyPath +
83 main.multiovs,
84 main.Mininet1.home,
85 direction="to" )
86
87 if main.CLIs:
88 stepResult = main.TRUE
89 else:
90 main.log.error( "Did not properly created list of " +
91 "ONOS CLI handle" )
92 stepResult = main.FALSE
93
kelvin-onlab1d381fe2015-07-14 16:24:56 -070094 utilities.assert_equals( expect=main.TRUE,
95 actual=stepResult,
96 onpass="Successfully construct " +
97 "test variables ",
98 onfail="Failed to construct test variables" )
99
100 if gitPull == 'True':
101 main.step( "Building ONOS in " + gitBranch + " branch" )
102 onosBuildResult = main.startUp.onosBuild( main, gitBranch )
103 stepResult = onosBuildResult
104 utilities.assert_equals( expect=main.TRUE,
105 actual=stepResult,
106 onpass="Successfully compiled " +
107 "latest ONOS",
108 onfail="Failed to compile " +
109 "latest ONOS" )
110 else:
111 main.log.warn( "Did not pull new code so skipping mvn " +
112 "clean install" )
113
GlennRC632e2892015-10-19 18:58:41 -0700114
115 def CASE2( self, main):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700116 """
117 - Set up cell
118 - Create cell file
119 - Set cell file
120 - Verify cell file
121 - Kill ONOS process
122 - Uninstall ONOS cluster
123 - Verify ONOS start up
124 - Install ONOS cluster
125 - Connect to cli
126 """
127
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700128 main.case( "Starting up " + str( main.numCtrls ) +
129 " node(s) ONOS cluster" )
GlennRC632e2892015-10-19 18:58:41 -0700130 main.caseExplanation = "Set up ONOS with " + str( main.numCtrls ) +\
131 " node(s) ONOS cluster"
132
133
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700134
135 #kill off all onos processes
136 main.log.info( "Safety check, killing all ONOS processes" +
Jon Hall70b2ff42015-11-17 15:49:44 -0800137 " before initiating environment setup" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700138
GlennRC632e2892015-10-19 18:58:41 -0700139 for i in range( main.numCtrls ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700140 main.ONOSbench.onosDie( main.ONOSip[ i ] )
141
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700142 tempOnosIp = []
143 for i in range( main.numCtrls ):
144 tempOnosIp.append( main.ONOSip[i] )
145
146 main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
147 "temp", main.Mininet1.ip_address,
GlennRC632e2892015-10-19 18:58:41 -0700148 main.apps, tempOnosIp )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700149
150 main.step( "Apply cell to environment" )
151 cellResult = main.ONOSbench.setCell( "temp" )
152 verifyResult = main.ONOSbench.verifyCell()
153 stepResult = cellResult and verifyResult
154 utilities.assert_equals( expect=main.TRUE,
155 actual=stepResult,
156 onpass="Successfully applied cell to " + \
157 "environment",
158 onfail="Failed to apply cell to environment " )
159
160 main.step( "Creating ONOS package" )
161 packageResult = main.ONOSbench.onosPackage()
162 stepResult = packageResult
163 utilities.assert_equals( expect=main.TRUE,
164 actual=stepResult,
165 onpass="Successfully created ONOS package",
166 onfail="Failed to create ONOS package" )
167
GlennRC632e2892015-10-19 18:58:41 -0700168 time.sleep( main.startUpSleep )
169 main.step( "Uninstalling ONOS package" )
170 onosUninstallResult = main.TRUE
171 for ip in main.ONOSip:
172 onosUninstallResult = onosUninstallResult and \
173 main.ONOSbench.onosUninstall( nodeIp=ip )
174 stepResult = onosUninstallResult
175 utilities.assert_equals( expect=main.TRUE,
176 actual=stepResult,
177 onpass="Successfully uninstalled ONOS package",
178 onfail="Failed to uninstall ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700179
GlennRC632e2892015-10-19 18:58:41 -0700180 time.sleep( main.startUpSleep )
181 main.step( "Installing ONOS package" )
182 onosInstallResult = main.TRUE
183 for i in range( main.numCtrls ):
184 onosInstallResult = onosInstallResult and \
185 main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
186 stepResult = onosInstallResult
187 utilities.assert_equals( expect=main.TRUE,
188 actual=stepResult,
189 onpass="Successfully installed ONOS package",
190 onfail="Failed to install ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700191
GlennRC632e2892015-10-19 18:58:41 -0700192 time.sleep( main.startUpSleep )
193 main.step( "Starting ONOS service" )
194 stopResult = main.TRUE
195 startResult = main.TRUE
196 onosIsUp = main.TRUE
197
198 for i in range( main.numCtrls ):
199 onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
200 if onosIsUp == main.TRUE:
201 main.log.report( "ONOS instance is up and ready" )
202 else:
203 main.log.report( "ONOS instance may not be up, stop and " +
204 "start ONOS again " )
205
206 for i in range( main.numCtrls ):
207 stopResult = stopResult and \
208 main.ONOSbench.onosStop( main.ONOSip[ i ] )
209 for i in range( main.numCtrls ):
210 startResult = startResult and \
211 main.ONOSbench.onosStart( main.ONOSip[ i ] )
212 stepResult = onosIsUp and stopResult and startResult
213 utilities.assert_equals( expect=main.TRUE,
214 actual=stepResult,
215 onpass="ONOS service is ready",
216 onfail="ONOS service did not start properly" )
217
218 main.step( "Start ONOS cli" )
219 cliResult = main.TRUE
220 for i in range( main.numCtrls ):
221 cliResult = cliResult and \
222 main.CLIs[ i ].startOnosCli( main.ONOSip[ i ] )
223 main.activeNodes.append( i )
224 stepResult = cliResult
225 utilities.assert_equals( expect=main.TRUE,
226 actual=stepResult,
227 onpass="Successfully start ONOS cli",
228 onfail="Failed to start ONOS cli" )
229
230
231 def CASE10( self, main ):
232 """
GlennRC475f50d2015-10-23 15:01:09 -0700233 Starting up torus topology, pingall, and compare topo
GlennRC632e2892015-10-19 18:58:41 -0700234 """
GlennRC475f50d2015-10-23 15:01:09 -0700235 import json
236
237 main.case( "Starting up Mininet and verifying topology" )
238 main.caseExplanation = "Starting Mininet with a scalling topology and " +\
239 "comparing topology elements between Mininet and ONOS"
GlennRC632e2892015-10-19 18:58:41 -0700240
241 main.log.info( "Checking if mininet is already running" )
242 if len( main.topoScale ) < main.topoScaleSize:
243 main.log.info( "Mininet is already running. Stopping mininet." )
244 main.Mininet1.stopNet()
GlennRC475f50d2015-10-23 15:01:09 -0700245 time.sleep(main.stopMNSleep)
GlennRC632e2892015-10-19 18:58:41 -0700246 else:
247 main.log.info( "Mininet was not running" )
248
GlennRC475f50d2015-10-23 15:01:09 -0700249 if main.topoScale:
GlennRC90d43952015-10-27 11:36:15 -0700250 main.currScale = main.topoScale.pop(0)
GlennRC475f50d2015-10-23 15:01:09 -0700251 else: main.log.error( "topology scale is empty" )
GlennRC632e2892015-10-19 18:58:41 -0700252
GlennRC475f50d2015-10-23 15:01:09 -0700253
GlennRC90d43952015-10-27 11:36:15 -0700254 main.step( "Starting up TORUS %sx%s topology" % (main.currScale, main.currScale) )
GlennRC475f50d2015-10-23 15:01:09 -0700255
256 main.log.info( "Constructing Mininet command" )
257 mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs +\
GlennRC90d43952015-10-27 11:36:15 -0700258 " --switch ovsm --topo " + main.topoName + ","+ main.currScale + "," + main.currScale
GlennRC475f50d2015-10-23 15:01:09 -0700259
260 for i in range( main.numCtrls ):
261 mnCmd += " --controller remote,ip=" + main.ONOSip[ i ]
262
GlennRC632e2892015-10-19 18:58:41 -0700263 stepResult = main.Mininet1.startNet(mnCmd=mnCmd)
264 utilities.assert_equals( expect=main.TRUE,
265 actual=stepResult,
266 onpass=main.topoName +
267 " topology started successfully",
268 onfail=main.topoName +
269 " topology failed to start" )
270
GlennRC475f50d2015-10-23 15:01:09 -0700271 time.sleep( main.startMNSleep )
GlennRC632e2892015-10-19 18:58:41 -0700272
GlennRC475f50d2015-10-23 15:01:09 -0700273 main.step( "Pinging all hosts" )
274
275 for i in range(main.pingallAttempts):
276 pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
277 if not pingResult:
278 main.log.warn( "Pingall attempt: %s failed" % (i+1) )
279 time.sleep(main.pingallSleep)
280 else: break
GlennRC632e2892015-10-19 18:58:41 -0700281
282 utilities.assert_equals( expect=main.TRUE,
283 actual=pingResult,
284 onpass="Pingall successfull",
285 onfail="Pingall failed" )
286
GlennRC475f50d2015-10-23 15:01:09 -0700287 main.log.info( "Gathering topology information" )
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700288 devicesResults = main.TRUE
289 linksResults = main.TRUE
290 hostsResults = main.TRUE
GlennRC632e2892015-10-19 18:58:41 -0700291
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700292 devices = main.topo.getAllDevices( main )
293 hosts = main.topo.getAllHosts( main )
294 ports = main.topo.getAllPorts( main )
295 links = main.topo.getAllLinks( main )
296 clusters = main.topo.getAllClusters( main )
297
298 mnSwitches = main.Mininet1.getSwitches()
299 mnLinks = main.Mininet1.getLinks()
300 mnHosts = main.Mininet1.getHosts()
301
GlennRC632e2892015-10-19 18:58:41 -0700302 main.step( "Comparing MN topology to ONOS topology" )
303
304 for controller in range(len(main.activeNodes)):
305 controllerStr = str( main.activeNodes[controller] + 1 )
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700306 if devices[ controller ] and ports[ controller ] and\
307 "Error" not in devices[ controller ] and\
308 "Error" not in ports[ controller ]:
309
310 currentDevicesResult = main.Mininet1.compareSwitches(
311 mnSwitches,
312 json.loads( devices[ controller ] ),
313 json.loads( ports[ controller ] ) )
314 else:
315 currentDevicesResult = main.FALSE
316 utilities.assert_equals( expect=main.TRUE,
317 actual=currentDevicesResult,
318 onpass="ONOS" + controllerStr +
319 " Switches view is correct",
320 onfail="ONOS" + controllerStr +
321 " Switches view is incorrect" )
322
323 if links[ controller ] and "Error" not in links[ controller ]:
324 currentLinksResult = main.Mininet1.compareLinks(
325 mnSwitches, mnLinks,
326 json.loads( links[ controller ] ) )
327 else:
328 currentLinksResult = main.FALSE
329 utilities.assert_equals( expect=main.TRUE,
330 actual=currentLinksResult,
331 onpass="ONOS" + controllerStr +
332 " links view is correct",
333 onfail="ONOS" + controllerStr +
334 " links view is incorrect" )
335
336 if hosts[ controller ] or "Error" not in hosts[ controller ]:
337 currentHostsResult = main.Mininet1.compareHosts(
338 mnHosts,
339 json.loads( hosts[ controller ] ) )
340 else:
341 currentHostsResult = main.FALSE
GlennRC632e2892015-10-19 18:58:41 -0700342 utilities.assert_equals( expect=main.TRUE,
343 actual=currentHostsResult,
344 onpass="ONOS" + controllerStr +
345 " hosts exist in Mininet",
346 onfail="ONOS" + controllerStr +
347 " hosts don't match Mininet" )
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700348
GlennRC475f50d2015-10-23 15:01:09 -0700349 def CASE11( self, main ):
350 """
351 Pingall, and compare topo
352 """
353 import json
354
GlennRC90d43952015-10-27 11:36:15 -0700355 main.case( "Verifying topology: TORUS %sx%s" % (main.currScale, main.currScale) )
GlennRC475f50d2015-10-23 15:01:09 -0700356 main.caseExplanation = "Pinging all hosts andcomparing topology " +\
357 "elements between Mininet and ONOS"
358 main.step( "Pinging all hosts" )
359
360 for i in range(main.pingallAttempts):
361 pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
362 if not pingResult:
363 main.log.warn( "Pingall attempt: %s failed" % (i+1) )
364 time.sleep(main.pingallSleep)
365 else: break
366
367 utilities.assert_equals( expect=main.TRUE,
368 actual=pingResult,
369 onpass="Pingall successfull",
370 onfail="Pingall failed" )
371
372 main.log.info( "Gathering topology information" )
373 devicesResults = main.TRUE
374 linksResults = main.TRUE
375 hostsResults = main.TRUE
376
377 devices = main.topo.getAllDevices( main )
378 hosts = main.topo.getAllHosts( main )
379 ports = main.topo.getAllPorts( main )
380 links = main.topo.getAllLinks( main )
381 clusters = main.topo.getAllClusters( main )
382
383 mnSwitches = main.Mininet1.getSwitches()
384 mnLinks = main.Mininet1.getLinks()
385 mnHosts = main.Mininet1.getHosts()
386
387 main.step( "Comparing MN topology to ONOS topology" )
388
389 for controller in range(len(main.activeNodes)):
390 controllerStr = str( main.activeNodes[controller] + 1 )
391 if devices[ controller ] and ports[ controller ] and\
392 "Error" not in devices[ controller ] and\
393 "Error" not in ports[ controller ]:
394
395 currentDevicesResult = main.Mininet1.compareSwitches(
396 mnSwitches,
397 json.loads( devices[ controller ] ),
398 json.loads( ports[ controller ] ) )
399 else:
400 currentDevicesResult = main.FALSE
401 utilities.assert_equals( expect=main.TRUE,
402 actual=currentDevicesResult,
403 onpass="ONOS" + controllerStr +
404 " Switches view is correct",
405 onfail="ONOS" + controllerStr +
406 " Switches view is incorrect" )
407
408 if links[ controller ] and "Error" not in links[ controller ]:
409 currentLinksResult = main.Mininet1.compareLinks(
410 mnSwitches, mnLinks,
411 json.loads( links[ controller ] ) )
412 else:
413 currentLinksResult = main.FALSE
414 utilities.assert_equals( expect=main.TRUE,
415 actual=currentLinksResult,
416 onpass="ONOS" + controllerStr +
417 " links view is correct",
418 onfail="ONOS" + controllerStr +
419 " links view is incorrect" )
420
421 if hosts[ controller ] or "Error" not in hosts[ controller ]:
422 currentHostsResult = main.Mininet1.compareHosts(
423 mnHosts,
424 json.loads( hosts[ controller ] ) )
425 else:
426 currentHostsResult = main.FALSE
427 utilities.assert_equals( expect=main.TRUE,
428 actual=currentHostsResult,
429 onpass="ONOS" + controllerStr +
430 " hosts exist in Mininet",
431 onfail="ONOS" + controllerStr +
432 " hosts don't match Mininet" )
433
434
GlennRC632e2892015-10-19 18:58:41 -0700435 def CASE100( self, main ):
436 '''
GlennRC475f50d2015-10-23 15:01:09 -0700437 Balance masters, ping and bring third ONOS node down
GlennRC632e2892015-10-19 18:58:41 -0700438 '''
GlennRC475f50d2015-10-23 15:01:09 -0700439
GlennRC90d43952015-10-27 11:36:15 -0700440 main.case("Balancing Masters and bring ONOS node 3 down: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700441 main.caseExplanation = "Balance masters to make sure " +\
442 "each controller has some devices and " +\
443 "stop ONOS node 3 service. "
444
445 main.step( "Balancing Masters" )
446 stepResult = main.FALSE
447 if main.activeNodes:
GlennRC632e2892015-10-19 18:58:41 -0700448 controller = main.activeNodes[0]
449 stepResult = main.CLIs[controller].balanceMasters()
GlennRC475f50d2015-10-23 15:01:09 -0700450 else: main.log.error( "List of active nodes is empty" )
451
GlennRC632e2892015-10-19 18:58:41 -0700452 utilities.assert_equals( expect=main.TRUE,
453 actual=stepResult,
454 onpass="Balance masters was successfull",
455 onfail="Failed to balance masters")
456
457 time.sleep(main.balanceSleep)
458
GlennRC475f50d2015-10-23 15:01:09 -0700459 main.step( "Pinging all hosts" )
GlennRC632e2892015-10-19 18:58:41 -0700460
GlennRC475f50d2015-10-23 15:01:09 -0700461 for i in range(main.pingallAttempts):
462 pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
463 if not pingResult:
464 main.log.warn( "Pingall attempt: %s failed" % (i+1) )
465 time.sleep(main.pingallSleep)
466 else: break
467
468 utilities.assert_equals( expect=main.TRUE,
469 actual=pingResult,
470 onpass="Pingall successfull",
471 onfail="Pingall failed" )
472
GlennRCed2122e2015-10-21 14:38:46 -0700473 main.step( "Bringing down node 3" )
GlennRC632e2892015-10-19 18:58:41 -0700474
GlennRCed2122e2015-10-21 14:38:46 -0700475 # Always bring down the third node
476 main.deadNode = 2
GlennRC632e2892015-10-19 18:58:41 -0700477
GlennRCed2122e2015-10-21 14:38:46 -0700478 # Printing purposes
GlennRC632e2892015-10-19 18:58:41 -0700479 node = main.deadNode + 1
480
GlennRC632e2892015-10-19 18:58:41 -0700481 main.log.info( "Stopping node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700482 stepResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700483
GlennRC475f50d2015-10-23 15:01:09 -0700484 main.log.info( "Removing dead node from list of active nodes" )
485 main.activeNodes.pop( main.deadNode )
GlennRC632e2892015-10-19 18:58:41 -0700486
487 utilities.assert_equals( expect=main.TRUE,
488 actual=stepResult,
489 onpass="Successfully brought down onos node %s" % node,
490 onfail="Failed to bring down onos node %s" % node )
491
492 time.sleep(main.nodeDownSleep)
493
494
GlennRC475f50d2015-10-23 15:01:09 -0700495 def CASE200( self, main ):
GlennRC632e2892015-10-19 18:58:41 -0700496 '''
GlennRC475f50d2015-10-23 15:01:09 -0700497 Bring up onos node and balance masters
GlennRC632e2892015-10-19 18:58:41 -0700498 '''
GlennRC475f50d2015-10-23 15:01:09 -0700499
GlennRC90d43952015-10-27 11:36:15 -0700500 main.case("Bring ONOS node 3 up and balance masters: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700501 main.caseExplanation = "Bring node 3 back up and balance the masters"
GlennRC632e2892015-10-19 18:58:41 -0700502
503 node = main.deadNode + 1
504
505 main.log.info( "Starting node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700506 stepResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700507
508 main.log.info( "Starting onos cli" )
GlennRC475f50d2015-10-23 15:01:09 -0700509 stepResult = stepResult and main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700510
GlennRC475f50d2015-10-23 15:01:09 -0700511 main.log.info( "Adding previously dead node to list of active nodes" )
GlennRC632e2892015-10-19 18:58:41 -0700512 main.activeNodes.append( main.deadNode )
513
GlennRC632e2892015-10-19 18:58:41 -0700514 utilities.assert_equals( expect=main.TRUE,
515 actual=stepResult,
516 onpass="Successfully brought up onos node %s" % node,
517 onfail="Failed to bring up onos node %s" % node )
518
519
520 time.sleep(main.nodeUpSleep)
521
GlennRC475f50d2015-10-23 15:01:09 -0700522 main.step( "Balancing Masters" )
523 stepResult = main.FALSE
524 if main.activeNodes:
525 controller = main.activeNodes[0]
526 stepResult = main.CLIs[controller].balanceMasters()
527 else: main.log.error( "List of active nodes is empty" )
528
529 utilities.assert_equals( expect=main.TRUE,
530 actual=stepResult,
531 onpass="Balance masters was successfull",
532 onfail="Failed to balance masters")
533
534 time.sleep(main.balanceSleep)
535
536 for i in range(main.pingallAttempts):
537 pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
538 if not pingResult:
539 main.log.warn( "Pingall attempt: %s failed" % (i+1) )
540 time.sleep(main.pingallSleep)
541 else: break
542
GlennRC632e2892015-10-19 18:58:41 -0700543 def CASE1000( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700544 '''
545 Report errors/warnings/exceptions
546 '''
GlennRC475f50d2015-10-23 15:01:09 -0700547 main.case( "Checking logs for errors, warnings, and exceptions" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700548 main.log.info("Error report: \n" )
549 main.ONOSbench.logReport( main.ONOSip[ 0 ],
GlennRC475f50d2015-10-23 15:01:09 -0700550 [ "INFO",
551 "FOLLOWER",
552 "WARN",
553 "flow",
554 "ERROR",
555 "Except" ],
556 "s" )