blob: 7babb4964504687a812334e29a86fe1c891cda50 [file] [log] [blame]
kelvin-onlab1d381fe2015-07-14 16:24:56 -07001
2# Testing network scalability, this test suite scales up a network topology
3# using mininet and verifies ONOS stability
4
GlennRC1c5df3c2015-08-27 16:12:09 -07005class SCPFscaleTopo:
kelvin-onlab1d381fe2015-07-14 16:24:56 -07006
7 def __init__( self ):
8 self.default = ''
9
10 def CASE1( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -070011 import os
12 import imp
13
14 """
15 - Construct tests variables
16 - GIT ( optional )
17 - Checkout ONOS master branch
18 - Pull latest ONOS code
19 - Building ONOS ( optional )
20 - Install ONOS package
21 - Build ONOS package
22 """
23
GlennRC475f50d2015-10-23 15:01:09 -070024 main.case( "Constructing test variables" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070025 main.step( "Constructing test variables" )
26 stepResult = main.FALSE
27
GlennRC475f50d2015-10-23 15:01:09 -070028 main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
29 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
30 gitBranch = main.params[ 'GIT' ][ 'branch' ]
31 main.dependencyPath = main.testOnDirectory + \
32 main.params[ 'DEPENDENCY' ][ 'path' ]
33 main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
34 main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
35 main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] )
36 main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
37 main.topoScaleSize = len( main.topoScale )
38 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
39 wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
40 wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
41 main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] )
42 main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] )
43 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
GlennRC475f50d2015-10-23 15:01:09 -070044 main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
GlennRCe283c4b2016-01-07 13:04:10 -080045 main.nodeSleep = int( main.params[ 'SLEEP' ][ 'nodeSleep' ] )
GlennRC475f50d2015-10-23 15:01:09 -070046 main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] )
GlennRCe283c4b2016-01-07 13:04:10 -080047 main.MNSleep = int( main.params[ 'SLEEP' ][ 'MNsleep' ] )
YPZhang85024fc2016-02-09 16:59:27 -080048 main.pingTimeout = float( main.params[ 'TIMEOUT' ][ 'pingall' ] )
YPZhangacaaf422016-07-26 09:34:03 -070049 main.hostDiscover = main.params[ 'TOPOLOGY' ][ 'host' ]
50 main.hostDiscoverSleep = float( main.params['SLEEP']['host'] )
51 if main.hostDiscover == 'True':
52 main.hostDiscover = True
53 else:
54 main.hostDiscover = False
GlennRC475f50d2015-10-23 15:01:09 -070055 gitPull = main.params[ 'GIT' ][ 'pull' ]
56 main.homeDir = os.path.expanduser('~')
57 main.cellData = {} # for creating cell file
58 main.hostsData = {}
59 main.CLIs = []
60 main.ONOSip = []
61 main.activeNodes = []
62 main.ONOSip = main.ONOSbench.getOnosIps()
kelvin-onlab1d381fe2015-07-14 16:24:56 -070063
GlennRC475f50d2015-10-23 15:01:09 -070064 for i in range(main.numCtrls):
GlennRC632e2892015-10-19 18:58:41 -070065 main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
66
GlennRC475f50d2015-10-23 15:01:09 -070067 main.startUp = imp.load_source( wrapperFile1,
68 main.dependencyPath +
69 wrapperFile1 +
70 ".py" )
GlennRC475f50d2015-10-23 15:01:09 -070071 main.scaleTopoFunction = imp.load_source( wrapperFile2,
72 main.dependencyPath +
73 wrapperFile2 +
74 ".py" )
GlennRC475f50d2015-10-23 15:01:09 -070075 main.topo = imp.load_source( wrapperFile3,
76 main.dependencyPath +
77 wrapperFile3 +
78 ".py" )
GlennRC632e2892015-10-19 18:58:41 -070079 main.ONOSbench.scp( main.Mininet1,
80 main.dependencyPath +
81 main.multiovs,
82 main.Mininet1.home,
83 direction="to" )
84
85 if main.CLIs:
86 stepResult = main.TRUE
87 else:
88 main.log.error( "Did not properly created list of " +
89 "ONOS CLI handle" )
90 stepResult = main.FALSE
91
kelvin-onlab1d381fe2015-07-14 16:24:56 -070092 utilities.assert_equals( expect=main.TRUE,
93 actual=stepResult,
94 onpass="Successfully construct " +
95 "test variables ",
96 onfail="Failed to construct test variables" )
97
98 if gitPull == 'True':
99 main.step( "Building ONOS in " + gitBranch + " branch" )
100 onosBuildResult = main.startUp.onosBuild( main, gitBranch )
101 stepResult = onosBuildResult
102 utilities.assert_equals( expect=main.TRUE,
103 actual=stepResult,
104 onpass="Successfully compiled " +
105 "latest ONOS",
106 onfail="Failed to compile " +
107 "latest ONOS" )
108 else:
109 main.log.warn( "Did not pull new code so skipping mvn " +
110 "clean install" )
111
GlennRC632e2892015-10-19 18:58:41 -0700112
113 def CASE2( self, main):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700114 """
115 - Set up cell
116 - Create cell file
117 - Set cell file
118 - Verify cell file
119 - Kill ONOS process
120 - Uninstall ONOS cluster
121 - Verify ONOS start up
122 - Install ONOS cluster
123 - Connect to cli
124 """
YPZhangacaaf422016-07-26 09:34:03 -0700125 import time
YPZhang29c2d642016-06-22 16:15:19 -0700126 main.log.info( "Checking if mininet is already running" )
127 if len( main.topoScale ) < main.topoScaleSize:
128 main.log.info( "Mininet is already running. Stopping mininet." )
129 main.Mininet1.stopNet()
130 time.sleep(main.MNSleep)
131 else:
132 main.log.info( "Mininet was not running" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700133
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700134 main.case( "Starting up " + str( main.numCtrls ) +
135 " node(s) ONOS cluster" )
GlennRC632e2892015-10-19 18:58:41 -0700136 main.caseExplanation = "Set up ONOS with " + str( main.numCtrls ) +\
137 " node(s) ONOS cluster"
138
139
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700140
141 #kill off all onos processes
142 main.log.info( "Safety check, killing all ONOS processes" +
Jon Hall70b2ff42015-11-17 15:49:44 -0800143 " before initiating environment setup" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700144
GlennRC632e2892015-10-19 18:58:41 -0700145 for i in range( main.numCtrls ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700146 main.ONOSbench.onosDie( main.ONOSip[ i ] )
147
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700148 tempOnosIp = []
149 for i in range( main.numCtrls ):
150 tempOnosIp.append( main.ONOSip[i] )
151
152 main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
153 "temp", main.Mininet1.ip_address,
GlennRC632e2892015-10-19 18:58:41 -0700154 main.apps, tempOnosIp )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700155
156 main.step( "Apply cell to environment" )
157 cellResult = main.ONOSbench.setCell( "temp" )
158 verifyResult = main.ONOSbench.verifyCell()
159 stepResult = cellResult and verifyResult
160 utilities.assert_equals( expect=main.TRUE,
161 actual=stepResult,
162 onpass="Successfully applied cell to " + \
163 "environment",
164 onfail="Failed to apply cell to environment " )
165
166 main.step( "Creating ONOS package" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700167 packageResult = main.ONOSbench.buckBuild()
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700168 stepResult = packageResult
169 utilities.assert_equals( expect=main.TRUE,
170 actual=stepResult,
171 onpass="Successfully created ONOS package",
172 onfail="Failed to create ONOS package" )
173
GlennRC632e2892015-10-19 18:58:41 -0700174 time.sleep( main.startUpSleep )
175 main.step( "Uninstalling ONOS package" )
176 onosUninstallResult = main.TRUE
177 for ip in main.ONOSip:
178 onosUninstallResult = onosUninstallResult and \
179 main.ONOSbench.onosUninstall( nodeIp=ip )
180 stepResult = onosUninstallResult
181 utilities.assert_equals( expect=main.TRUE,
182 actual=stepResult,
183 onpass="Successfully uninstalled ONOS package",
184 onfail="Failed to uninstall ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700185
GlennRC632e2892015-10-19 18:58:41 -0700186 time.sleep( main.startUpSleep )
187 main.step( "Installing ONOS package" )
188 onosInstallResult = main.TRUE
189 for i in range( main.numCtrls ):
190 onosInstallResult = onosInstallResult and \
191 main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
192 stepResult = onosInstallResult
193 utilities.assert_equals( expect=main.TRUE,
194 actual=stepResult,
195 onpass="Successfully installed ONOS package",
196 onfail="Failed to install ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700197
GlennRC632e2892015-10-19 18:58:41 -0700198 time.sleep( main.startUpSleep )
199 main.step( "Starting ONOS service" )
200 stopResult = main.TRUE
201 startResult = main.TRUE
202 onosIsUp = main.TRUE
203
204 for i in range( main.numCtrls ):
205 onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
206 if onosIsUp == main.TRUE:
207 main.log.report( "ONOS instance is up and ready" )
208 else:
209 main.log.report( "ONOS instance may not be up, stop and " +
210 "start ONOS again " )
211
212 for i in range( main.numCtrls ):
213 stopResult = stopResult and \
214 main.ONOSbench.onosStop( main.ONOSip[ i ] )
215 for i in range( main.numCtrls ):
216 startResult = startResult and \
217 main.ONOSbench.onosStart( main.ONOSip[ i ] )
218 stepResult = onosIsUp and stopResult and startResult
219 utilities.assert_equals( expect=main.TRUE,
220 actual=stepResult,
221 onpass="ONOS service is ready",
222 onfail="ONOS service did not start properly" )
223
224 main.step( "Start ONOS cli" )
225 cliResult = main.TRUE
YPZhang29c2d642016-06-22 16:15:19 -0700226 main.activeNodes = []
GlennRC632e2892015-10-19 18:58:41 -0700227 for i in range( main.numCtrls ):
228 cliResult = cliResult and \
229 main.CLIs[ i ].startOnosCli( main.ONOSip[ i ] )
230 main.activeNodes.append( i )
231 stepResult = cliResult
232 utilities.assert_equals( expect=main.TRUE,
233 actual=stepResult,
234 onpass="Successfully start ONOS cli",
235 onfail="Failed to start ONOS cli" )
YPZhang29c2d642016-06-22 16:15:19 -0700236 time.sleep( main.startUpSleep )
GlennRC632e2892015-10-19 18:58:41 -0700237
238 def CASE10( self, main ):
239 """
YPZhang85024fc2016-02-09 16:59:27 -0800240 Starting up torus topology
GlennRC632e2892015-10-19 18:58:41 -0700241 """
GlennRC475f50d2015-10-23 15:01:09 -0700242
243 main.case( "Starting up Mininet and verifying topology" )
244 main.caseExplanation = "Starting Mininet with a scalling topology and " +\
245 "comparing topology elements between Mininet and ONOS"
GlennRC475f50d2015-10-23 15:01:09 -0700246 if main.topoScale:
GlennRC90d43952015-10-27 11:36:15 -0700247 main.currScale = main.topoScale.pop(0)
GlennRC475f50d2015-10-23 15:01:09 -0700248 else: main.log.error( "topology scale is empty" )
GlennRC90d43952015-10-27 11:36:15 -0700249 main.step( "Starting up TORUS %sx%s topology" % (main.currScale, main.currScale) )
GlennRC475f50d2015-10-23 15:01:09 -0700250
251 main.log.info( "Constructing Mininet command" )
YPZhangacaaf422016-07-26 09:34:03 -0700252 mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs + \
253 " --switch ovsm --topo " + main.topoName + "," + main.currScale + "," + main.currScale
GlennRC475f50d2015-10-23 15:01:09 -0700254 for i in range( main.numCtrls ):
255 mnCmd += " --controller remote,ip=" + main.ONOSip[ i ]
256
GlennRC632e2892015-10-19 18:58:41 -0700257 stepResult = main.Mininet1.startNet(mnCmd=mnCmd)
258 utilities.assert_equals( expect=main.TRUE,
259 actual=stepResult,
260 onpass=main.topoName +
261 " topology started successfully",
262 onfail=main.topoName +
263 " topology failed to start" )
264
GlennRCe283c4b2016-01-07 13:04:10 -0800265 time.sleep( main.MNSleep )
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700266
GlennRC475f50d2015-10-23 15:01:09 -0700267 def CASE11( self, main ):
268 """
YPZhangacaaf422016-07-26 09:34:03 -0700269 Compare topo, and sending Arping package
YPZhang85024fc2016-02-09 16:59:27 -0800270 if the topology is same, then Pass.
GlennRC475f50d2015-10-23 15:01:09 -0700271 """
272 import json
YPZhangacaaf422016-07-26 09:34:03 -0700273 import time
GlennRC475f50d2015-10-23 15:01:09 -0700274
GlennRC90d43952015-10-27 11:36:15 -0700275 main.case( "Verifying topology: TORUS %sx%s" % (main.currScale, main.currScale) )
YPZhang85024fc2016-02-09 16:59:27 -0800276 main.caseExplanation = "Pinging all hosts and comparing topology " +\
GlennRC475f50d2015-10-23 15:01:09 -0700277 "elements between Mininet and ONOS"
GlennRCe283c4b2016-01-07 13:04:10 -0800278
GlennRC475f50d2015-10-23 15:01:09 -0700279 main.log.info( "Gathering topology information" )
YPZhang85024fc2016-02-09 16:59:27 -0800280 time.sleep( main.MNSleep )
YPZhang85024fc2016-02-09 16:59:27 -0800281 stepResult = main.TRUE
GlennRC475f50d2015-10-23 15:01:09 -0700282 main.step( "Comparing MN topology to ONOS topology" )
GlennRC475f50d2015-10-23 15:01:09 -0700283
YPZhang81a7d4e2016-04-18 13:10:17 -0700284 compareRetry=0
285 while compareRetry <3:
286 #While loop for retry
287 devices = main.topo.getAllDevices( main )
YPZhang81a7d4e2016-04-18 13:10:17 -0700288 ports = main.topo.getAllPorts( main )
289 links = main.topo.getAllLinks( main)
YPZhang81a7d4e2016-04-18 13:10:17 -0700290 mnSwitches = main.Mininet1.getSwitches()
291 mnLinks = main.Mininet1.getLinks(timeout=180)
GlennRC475f50d2015-10-23 15:01:09 -0700292
YPZhang81a7d4e2016-04-18 13:10:17 -0700293 for controller in range(len(main.activeNodes)):
YPZhangacaaf422016-07-26 09:34:03 -0700294 # controllerStr = str( main.activeNodes[controller] + 1 )
295 if devices[ controller ] and ports[ controller ] and \
296 "Error" not in devices[ controller ] and \
297 "Error" not in ports[ controller ]:
YPZhang81a7d4e2016-04-18 13:10:17 -0700298 currentDevicesResult = main.Mininet1.compareSwitches(
299 mnSwitches,
300 json.loads( devices[ controller ] ),
301 json.loads( ports[ controller ] ) )
302 else:
303 currentDevicesResult = main.FALSE
304
305 if links[ controller ] and "Error" not in links[ controller ]:
306 currentLinksResult = main.Mininet1.compareLinks(
307 mnSwitches, mnLinks,
308 json.loads( links[ controller ] ) )
309 else:
310 currentLinksResult = main.FALSE
311
YPZhangacaaf422016-07-26 09:34:03 -0700312 stepResult = stepResult and currentDevicesResult and currentLinksResult
YPZhang81a7d4e2016-04-18 13:10:17 -0700313 if stepResult:
314 break
315 compareRetry += 1
YPZhangacaaf422016-07-26 09:34:03 -0700316 utilities.assert_equals(expect=main.TRUE,
317 actual=stepResult,
318 onpass=" Topology match Mininet",
319 onfail="ONOS Topology doesn't match Mininet")
YPZhang81a7d4e2016-04-18 13:10:17 -0700320
YPZhangacaaf422016-07-26 09:34:03 -0700321 if stepResult:
322 if main.hostDiscover:
323 hostList = []
324 for i in range( 1, int( main.currScale ) + 1 ):
325 for j in range( 1, int( main.currScale ) + 1) :
326 # Generate host list
327 hoststr = "h" + str(i) + "x" + str(j)
328 hostList.append(hoststr)
329 for i in range( len(hostList) ):
330 totalHost = main.topo.sendArpPackage( main, hostList[i] )
331 time.sleep( main.hostDiscoverSleep )
332 if totalHost < 0:
333 # if totalHost less than 0 which means dependence function has exception.
334 main.log.info( "Error when discover host!" )
335 break
336 if totalHost == int( main.currScale ) * int( main.currScale ):
337 main.log.info( "Discovered all hosts" )
You Wang7bb9c462016-08-10 14:18:16 -0700338 stepResult = stepResult and main.TRUE
YPZhangacaaf422016-07-26 09:34:03 -0700339 else:
340 main.log.warn( "Some hosts ware not discovered by ONOS... Topology doesn't match!" )
You Wang7bb9c462016-08-10 14:18:16 -0700341 stepResult = main.FALSE
YPZhangacaaf422016-07-26 09:34:03 -0700342 utilities.assert_equals(expect=main.TRUE,
343 actual=stepResult,
344 onpass=" Topology match Mininet",
345 onfail="ONOS Topology doesn't match Mininet")
346 main.log.info( "Finished this iteration, continue to scale next topology." )
YPZhang81a7d4e2016-04-18 13:10:17 -0700347 else:
YPZhangacaaf422016-07-26 09:34:03 -0700348 main.log.info( "Clean up and exit TestON. Finished this test." )
349 main.cleanup()
350 main.exit()
GlennRC475f50d2015-10-23 15:01:09 -0700351
GlennRC632e2892015-10-19 18:58:41 -0700352 def CASE100( self, main ):
353 '''
YPZhang81a7d4e2016-04-18 13:10:17 -0700354 Bring Down node 3
GlennRC632e2892015-10-19 18:58:41 -0700355 '''
GlennRC475f50d2015-10-23 15:01:09 -0700356
YPZhangacaaf422016-07-26 09:34:03 -0700357 main.case("Bring ONOS node 3 down: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700358 main.caseExplanation = "Balance masters to make sure " +\
359 "each controller has some devices and " +\
360 "stop ONOS node 3 service. "
361
GlennRC475f50d2015-10-23 15:01:09 -0700362 stepResult = main.FALSE
GlennRCed2122e2015-10-21 14:38:46 -0700363 main.step( "Bringing down node 3" )
GlennRCed2122e2015-10-21 14:38:46 -0700364 # Always bring down the third node
365 main.deadNode = 2
GlennRCed2122e2015-10-21 14:38:46 -0700366 # Printing purposes
GlennRC632e2892015-10-19 18:58:41 -0700367 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700368 main.log.info( "Stopping node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700369 stepResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] )
GlennRC475f50d2015-10-23 15:01:09 -0700370 main.log.info( "Removing dead node from list of active nodes" )
371 main.activeNodes.pop( main.deadNode )
GlennRC632e2892015-10-19 18:58:41 -0700372
YPZhang77badfc2016-03-09 10:28:59 -0800373 utilities.assert_equals( expect=main.TRUE,
374 actual=stepResult,
375 onpass="Successfully bring down node 3",
376 onfail="Failed to bring down node 3" )
GlennRC632e2892015-10-19 18:58:41 -0700377
GlennRC475f50d2015-10-23 15:01:09 -0700378 def CASE200( self, main ):
GlennRC632e2892015-10-19 18:58:41 -0700379 '''
YPZhangacaaf422016-07-26 09:34:03 -0700380 Bring up onos node
GlennRC632e2892015-10-19 18:58:41 -0700381 '''
GlennRC475f50d2015-10-23 15:01:09 -0700382
YPZhangacaaf422016-07-26 09:34:03 -0700383 main.case("Bring ONOS node 3 up: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700384 main.caseExplanation = "Bring node 3 back up and balance the masters"
GlennRC632e2892015-10-19 18:58:41 -0700385
386 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700387 main.log.info( "Starting node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700388 stepResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700389 main.log.info( "Starting onos cli" )
GlennRC475f50d2015-10-23 15:01:09 -0700390 stepResult = stepResult and main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700391
GlennRC475f50d2015-10-23 15:01:09 -0700392 main.log.info( "Adding previously dead node to list of active nodes" )
GlennRC632e2892015-10-19 18:58:41 -0700393 main.activeNodes.append( main.deadNode )
394
GlennRC632e2892015-10-19 18:58:41 -0700395 utilities.assert_equals( expect=main.TRUE,
396 actual=stepResult,
397 onpass="Successfully brought up onos node %s" % node,
398 onfail="Failed to bring up onos node %s" % node )
399
400
GlennRCe283c4b2016-01-07 13:04:10 -0800401 time.sleep(main.nodeSleep)
402
403 def CASE300( self, main ):
404 '''
405
406 Balancing Masters
407 '''
YPZhang85024fc2016-02-09 16:59:27 -0800408 time.sleep(main.balanceSleep)
GlennRC475f50d2015-10-23 15:01:09 -0700409 main.step( "Balancing Masters" )
GlennRCe283c4b2016-01-07 13:04:10 -0800410
GlennRC475f50d2015-10-23 15:01:09 -0700411 stepResult = main.FALSE
412 if main.activeNodes:
413 controller = main.activeNodes[0]
YPZhang924ccfe2016-01-26 14:17:30 -0800414 stepResult = utilities.retry( main.CLIs[controller].balanceMasters,
415 main.FALSE,
416 [],
417 sleep=3,
418 attempts=3 )
419
GlennRCe283c4b2016-01-07 13:04:10 -0800420 else:
421 main.log.error( "List of active nodes is empty" )
GlennRC475f50d2015-10-23 15:01:09 -0700422 utilities.assert_equals( expect=main.TRUE,
423 actual=stepResult,
424 onpass="Balance masters was successfull",
425 onfail="Failed to balance masters")
GlennRC475f50d2015-10-23 15:01:09 -0700426 time.sleep(main.balanceSleep)
427
GlennRC632e2892015-10-19 18:58:41 -0700428 def CASE1000( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700429 '''
430 Report errors/warnings/exceptions
431 '''
GlennRC475f50d2015-10-23 15:01:09 -0700432 main.case( "Checking logs for errors, warnings, and exceptions" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700433 main.log.info("Error report: \n" )
434 main.ONOSbench.logReport( main.ONOSip[ 0 ],
GlennRC475f50d2015-10-23 15:01:09 -0700435 [ "INFO",
436 "FOLLOWER",
437 "WARN",
438 "flow",
439 "ERROR",
440 "Except" ],
441 "s" )