blob: ae197a6a3fe6ce23d878e46eea0384eee1984ec2 [file] [log] [blame]
kelvin-onlab1d381fe2015-07-14 16:24:56 -07001
2# Testing network scalability, this test suite scales up a network topology
3# using mininet and verifies ONOS stability
4
GlennRC1c5df3c2015-08-27 16:12:09 -07005class SCPFscaleTopo:
kelvin-onlab1d381fe2015-07-14 16:24:56 -07006
7 def __init__( self ):
8 self.default = ''
9
10 def CASE1( self, main ):
11 import time
12 import os
13 import imp
Jon Hallf632d202015-07-30 15:45:11 -070014 import re
kelvin-onlab1d381fe2015-07-14 16:24:56 -070015
16 """
17 - Construct tests variables
18 - GIT ( optional )
19 - Checkout ONOS master branch
20 - Pull latest ONOS code
21 - Building ONOS ( optional )
22 - Install ONOS package
23 - Build ONOS package
24 """
25
GlennRC475f50d2015-10-23 15:01:09 -070026 main.case( "Constructing test variables" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070027 main.step( "Constructing test variables" )
28 stepResult = main.FALSE
29
GlennRC475f50d2015-10-23 15:01:09 -070030 main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
31 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
32 gitBranch = main.params[ 'GIT' ][ 'branch' ]
33 main.dependencyPath = main.testOnDirectory + \
34 main.params[ 'DEPENDENCY' ][ 'path' ]
35 main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
36 main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
37 main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] )
38 main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
39 main.topoScaleSize = len( main.topoScale )
40 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
41 wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
42 wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
43 main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] )
44 main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] )
45 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
GlennRC475f50d2015-10-23 15:01:09 -070046 main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
GlennRCe283c4b2016-01-07 13:04:10 -080047 main.nodeSleep = int( main.params[ 'SLEEP' ][ 'nodeSleep' ] )
GlennRC475f50d2015-10-23 15:01:09 -070048 main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] )
GlennRCe283c4b2016-01-07 13:04:10 -080049 main.MNSleep = int( main.params[ 'SLEEP' ][ 'MNsleep' ] )
GlennRC475f50d2015-10-23 15:01:09 -070050 main.pingTimeout = int( main.params[ 'TIMEOUT' ][ 'pingall' ] )
51 gitPull = main.params[ 'GIT' ][ 'pull' ]
52 main.homeDir = os.path.expanduser('~')
53 main.cellData = {} # for creating cell file
54 main.hostsData = {}
55 main.CLIs = []
56 main.ONOSip = []
57 main.activeNodes = []
58 main.ONOSip = main.ONOSbench.getOnosIps()
kelvin-onlab1d381fe2015-07-14 16:24:56 -070059
GlennRC475f50d2015-10-23 15:01:09 -070060 for i in range(main.numCtrls):
GlennRC632e2892015-10-19 18:58:41 -070061 main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
62
GlennRC475f50d2015-10-23 15:01:09 -070063 main.startUp = imp.load_source( wrapperFile1,
64 main.dependencyPath +
65 wrapperFile1 +
66 ".py" )
GlennRC632e2892015-10-19 18:58:41 -070067
GlennRC475f50d2015-10-23 15:01:09 -070068 main.scaleTopoFunction = imp.load_source( wrapperFile2,
69 main.dependencyPath +
70 wrapperFile2 +
71 ".py" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070072
GlennRC475f50d2015-10-23 15:01:09 -070073 main.topo = imp.load_source( wrapperFile3,
74 main.dependencyPath +
75 wrapperFile3 +
76 ".py" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070077
GlennRC632e2892015-10-19 18:58:41 -070078 main.ONOSbench.scp( main.Mininet1,
79 main.dependencyPath +
80 main.multiovs,
81 main.Mininet1.home,
82 direction="to" )
83
84 if main.CLIs:
85 stepResult = main.TRUE
86 else:
87 main.log.error( "Did not properly created list of " +
88 "ONOS CLI handle" )
89 stepResult = main.FALSE
90
kelvin-onlab1d381fe2015-07-14 16:24:56 -070091 utilities.assert_equals( expect=main.TRUE,
92 actual=stepResult,
93 onpass="Successfully construct " +
94 "test variables ",
95 onfail="Failed to construct test variables" )
96
97 if gitPull == 'True':
98 main.step( "Building ONOS in " + gitBranch + " branch" )
99 onosBuildResult = main.startUp.onosBuild( main, gitBranch )
100 stepResult = onosBuildResult
101 utilities.assert_equals( expect=main.TRUE,
102 actual=stepResult,
103 onpass="Successfully compiled " +
104 "latest ONOS",
105 onfail="Failed to compile " +
106 "latest ONOS" )
107 else:
108 main.log.warn( "Did not pull new code so skipping mvn " +
109 "clean install" )
110
GlennRC632e2892015-10-19 18:58:41 -0700111
112 def CASE2( self, main):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700113 """
114 - Set up cell
115 - Create cell file
116 - Set cell file
117 - Verify cell file
118 - Kill ONOS process
119 - Uninstall ONOS cluster
120 - Verify ONOS start up
121 - Install ONOS cluster
122 - Connect to cli
123 """
124
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700125 main.case( "Starting up " + str( main.numCtrls ) +
126 " node(s) ONOS cluster" )
GlennRC632e2892015-10-19 18:58:41 -0700127 main.caseExplanation = "Set up ONOS with " + str( main.numCtrls ) +\
128 " node(s) ONOS cluster"
129
130
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700131
132 #kill off all onos processes
133 main.log.info( "Safety check, killing all ONOS processes" +
Jon Hall70b2ff42015-11-17 15:49:44 -0800134 " before initiating environment setup" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700135
GlennRC632e2892015-10-19 18:58:41 -0700136 for i in range( main.numCtrls ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700137 main.ONOSbench.onosDie( main.ONOSip[ i ] )
138
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700139 tempOnosIp = []
140 for i in range( main.numCtrls ):
141 tempOnosIp.append( main.ONOSip[i] )
142
143 main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
144 "temp", main.Mininet1.ip_address,
GlennRC632e2892015-10-19 18:58:41 -0700145 main.apps, tempOnosIp )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700146
147 main.step( "Apply cell to environment" )
148 cellResult = main.ONOSbench.setCell( "temp" )
149 verifyResult = main.ONOSbench.verifyCell()
150 stepResult = cellResult and verifyResult
151 utilities.assert_equals( expect=main.TRUE,
152 actual=stepResult,
153 onpass="Successfully applied cell to " + \
154 "environment",
155 onfail="Failed to apply cell to environment " )
156
157 main.step( "Creating ONOS package" )
158 packageResult = main.ONOSbench.onosPackage()
159 stepResult = packageResult
160 utilities.assert_equals( expect=main.TRUE,
161 actual=stepResult,
162 onpass="Successfully created ONOS package",
163 onfail="Failed to create ONOS package" )
164
GlennRC632e2892015-10-19 18:58:41 -0700165 time.sleep( main.startUpSleep )
166 main.step( "Uninstalling ONOS package" )
167 onosUninstallResult = main.TRUE
168 for ip in main.ONOSip:
169 onosUninstallResult = onosUninstallResult and \
170 main.ONOSbench.onosUninstall( nodeIp=ip )
171 stepResult = onosUninstallResult
172 utilities.assert_equals( expect=main.TRUE,
173 actual=stepResult,
174 onpass="Successfully uninstalled ONOS package",
175 onfail="Failed to uninstall ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700176
GlennRC632e2892015-10-19 18:58:41 -0700177 time.sleep( main.startUpSleep )
178 main.step( "Installing ONOS package" )
179 onosInstallResult = main.TRUE
180 for i in range( main.numCtrls ):
181 onosInstallResult = onosInstallResult and \
182 main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
183 stepResult = onosInstallResult
184 utilities.assert_equals( expect=main.TRUE,
185 actual=stepResult,
186 onpass="Successfully installed ONOS package",
187 onfail="Failed to install ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700188
GlennRC632e2892015-10-19 18:58:41 -0700189 time.sleep( main.startUpSleep )
190 main.step( "Starting ONOS service" )
191 stopResult = main.TRUE
192 startResult = main.TRUE
193 onosIsUp = main.TRUE
194
195 for i in range( main.numCtrls ):
196 onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
197 if onosIsUp == main.TRUE:
198 main.log.report( "ONOS instance is up and ready" )
199 else:
200 main.log.report( "ONOS instance may not be up, stop and " +
201 "start ONOS again " )
202
203 for i in range( main.numCtrls ):
204 stopResult = stopResult and \
205 main.ONOSbench.onosStop( main.ONOSip[ i ] )
206 for i in range( main.numCtrls ):
207 startResult = startResult and \
208 main.ONOSbench.onosStart( main.ONOSip[ i ] )
209 stepResult = onosIsUp and stopResult and startResult
210 utilities.assert_equals( expect=main.TRUE,
211 actual=stepResult,
212 onpass="ONOS service is ready",
213 onfail="ONOS service did not start properly" )
214
215 main.step( "Start ONOS cli" )
216 cliResult = main.TRUE
217 for i in range( main.numCtrls ):
218 cliResult = cliResult and \
219 main.CLIs[ i ].startOnosCli( main.ONOSip[ i ] )
220 main.activeNodes.append( i )
221 stepResult = cliResult
222 utilities.assert_equals( expect=main.TRUE,
223 actual=stepResult,
224 onpass="Successfully start ONOS cli",
225 onfail="Failed to start ONOS cli" )
226
227
228 def CASE10( self, main ):
229 """
GlennRC475f50d2015-10-23 15:01:09 -0700230 Starting up torus topology, pingall, and compare topo
GlennRC632e2892015-10-19 18:58:41 -0700231 """
GlennRC475f50d2015-10-23 15:01:09 -0700232 import json
233
234 main.case( "Starting up Mininet and verifying topology" )
235 main.caseExplanation = "Starting Mininet with a scalling topology and " +\
236 "comparing topology elements between Mininet and ONOS"
GlennRC632e2892015-10-19 18:58:41 -0700237
238 main.log.info( "Checking if mininet is already running" )
239 if len( main.topoScale ) < main.topoScaleSize:
240 main.log.info( "Mininet is already running. Stopping mininet." )
241 main.Mininet1.stopNet()
GlennRCe283c4b2016-01-07 13:04:10 -0800242 time.sleep(main.MNSleep)
GlennRC632e2892015-10-19 18:58:41 -0700243 else:
244 main.log.info( "Mininet was not running" )
245
GlennRC475f50d2015-10-23 15:01:09 -0700246 if main.topoScale:
GlennRC90d43952015-10-27 11:36:15 -0700247 main.currScale = main.topoScale.pop(0)
GlennRC475f50d2015-10-23 15:01:09 -0700248 else: main.log.error( "topology scale is empty" )
GlennRC632e2892015-10-19 18:58:41 -0700249
GlennRC475f50d2015-10-23 15:01:09 -0700250
GlennRC90d43952015-10-27 11:36:15 -0700251 main.step( "Starting up TORUS %sx%s topology" % (main.currScale, main.currScale) )
GlennRC475f50d2015-10-23 15:01:09 -0700252
253 main.log.info( "Constructing Mininet command" )
254 mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs +\
GlennRC90d43952015-10-27 11:36:15 -0700255 " --switch ovsm --topo " + main.topoName + ","+ main.currScale + "," + main.currScale
GlennRC475f50d2015-10-23 15:01:09 -0700256
257 for i in range( main.numCtrls ):
258 mnCmd += " --controller remote,ip=" + main.ONOSip[ i ]
259
GlennRC632e2892015-10-19 18:58:41 -0700260 stepResult = main.Mininet1.startNet(mnCmd=mnCmd)
261 utilities.assert_equals( expect=main.TRUE,
262 actual=stepResult,
263 onpass=main.topoName +
264 " topology started successfully",
265 onfail=main.topoName +
266 " topology failed to start" )
267
GlennRCe283c4b2016-01-07 13:04:10 -0800268 time.sleep( main.MNSleep )
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700269
GlennRC475f50d2015-10-23 15:01:09 -0700270 def CASE11( self, main ):
271 """
272 Pingall, and compare topo
273 """
274 import json
275
GlennRC90d43952015-10-27 11:36:15 -0700276 main.case( "Verifying topology: TORUS %sx%s" % (main.currScale, main.currScale) )
GlennRC475f50d2015-10-23 15:01:09 -0700277 main.caseExplanation = "Pinging all hosts andcomparing topology " +\
278 "elements between Mininet and ONOS"
GlennRCe283c4b2016-01-07 13:04:10 -0800279
GlennRC475f50d2015-10-23 15:01:09 -0700280 main.step( "Pinging all hosts" )
GlennRCe283c4b2016-01-07 13:04:10 -0800281 pingResult = utilities.retry( main.Mininet1.pingall,
282 main.FALSE,
283 [main.pingTimeout],
284 sleep=main.pingallSleep,
285 attempts=main.pingallAttempts )
GlennRC475f50d2015-10-23 15:01:09 -0700286 utilities.assert_equals( expect=main.TRUE,
287 actual=pingResult,
288 onpass="Pingall successfull",
289 onfail="Pingall failed" )
290
291 main.log.info( "Gathering topology information" )
292 devicesResults = main.TRUE
293 linksResults = main.TRUE
294 hostsResults = main.TRUE
GlennRC475f50d2015-10-23 15:01:09 -0700295 devices = main.topo.getAllDevices( main )
296 hosts = main.topo.getAllHosts( main )
297 ports = main.topo.getAllPorts( main )
298 links = main.topo.getAllLinks( main )
299 clusters = main.topo.getAllClusters( main )
GlennRC475f50d2015-10-23 15:01:09 -0700300 mnSwitches = main.Mininet1.getSwitches()
301 mnLinks = main.Mininet1.getLinks()
302 mnHosts = main.Mininet1.getHosts()
303
304 main.step( "Comparing MN topology to ONOS topology" )
GlennRC475f50d2015-10-23 15:01:09 -0700305 for controller in range(len(main.activeNodes)):
306 controllerStr = str( main.activeNodes[controller] + 1 )
307 if devices[ controller ] and ports[ controller ] and\
308 "Error" not in devices[ controller ] and\
309 "Error" not in ports[ controller ]:
310
311 currentDevicesResult = main.Mininet1.compareSwitches(
312 mnSwitches,
313 json.loads( devices[ controller ] ),
314 json.loads( ports[ controller ] ) )
315 else:
316 currentDevicesResult = main.FALSE
317 utilities.assert_equals( expect=main.TRUE,
318 actual=currentDevicesResult,
319 onpass="ONOS" + controllerStr +
320 " Switches view is correct",
321 onfail="ONOS" + controllerStr +
322 " Switches view is incorrect" )
323
324 if links[ controller ] and "Error" not in links[ controller ]:
325 currentLinksResult = main.Mininet1.compareLinks(
326 mnSwitches, mnLinks,
327 json.loads( links[ controller ] ) )
328 else:
329 currentLinksResult = main.FALSE
330 utilities.assert_equals( expect=main.TRUE,
331 actual=currentLinksResult,
332 onpass="ONOS" + controllerStr +
333 " links view is correct",
334 onfail="ONOS" + controllerStr +
335 " links view is incorrect" )
336
337 if hosts[ controller ] or "Error" not in hosts[ controller ]:
338 currentHostsResult = main.Mininet1.compareHosts(
339 mnHosts,
340 json.loads( hosts[ controller ] ) )
341 else:
342 currentHostsResult = main.FALSE
343 utilities.assert_equals( expect=main.TRUE,
344 actual=currentHostsResult,
345 onpass="ONOS" + controllerStr +
346 " hosts exist in Mininet",
347 onfail="ONOS" + controllerStr +
348 " hosts don't match Mininet" )
349
350
GlennRC632e2892015-10-19 18:58:41 -0700351 def CASE100( self, main ):
352 '''
GlennRC475f50d2015-10-23 15:01:09 -0700353 Balance masters, ping and bring third ONOS node down
GlennRC632e2892015-10-19 18:58:41 -0700354 '''
GlennRC475f50d2015-10-23 15:01:09 -0700355
GlennRC90d43952015-10-27 11:36:15 -0700356 main.case("Balancing Masters and bring ONOS node 3 down: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700357 main.caseExplanation = "Balance masters to make sure " +\
358 "each controller has some devices and " +\
359 "stop ONOS node 3 service. "
360
GlennRC475f50d2015-10-23 15:01:09 -0700361 stepResult = main.FALSE
GlennRCed2122e2015-10-21 14:38:46 -0700362 main.step( "Bringing down node 3" )
GlennRCed2122e2015-10-21 14:38:46 -0700363 # Always bring down the third node
364 main.deadNode = 2
GlennRCed2122e2015-10-21 14:38:46 -0700365 # Printing purposes
GlennRC632e2892015-10-19 18:58:41 -0700366 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700367 main.log.info( "Stopping node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700368 stepResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] )
GlennRC475f50d2015-10-23 15:01:09 -0700369 main.log.info( "Removing dead node from list of active nodes" )
370 main.activeNodes.pop( main.deadNode )
GlennRC632e2892015-10-19 18:58:41 -0700371
GlennRC632e2892015-10-19 18:58:41 -0700372
373
GlennRC475f50d2015-10-23 15:01:09 -0700374 def CASE200( self, main ):
GlennRC632e2892015-10-19 18:58:41 -0700375 '''
GlennRC475f50d2015-10-23 15:01:09 -0700376 Bring up onos node and balance masters
GlennRC632e2892015-10-19 18:58:41 -0700377 '''
GlennRC475f50d2015-10-23 15:01:09 -0700378
GlennRC90d43952015-10-27 11:36:15 -0700379 main.case("Bring ONOS node 3 up and balance masters: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700380 main.caseExplanation = "Bring node 3 back up and balance the masters"
GlennRC632e2892015-10-19 18:58:41 -0700381
382 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700383 main.log.info( "Starting node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700384 stepResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700385 main.log.info( "Starting onos cli" )
GlennRC475f50d2015-10-23 15:01:09 -0700386 stepResult = stepResult and main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700387
GlennRC475f50d2015-10-23 15:01:09 -0700388 main.log.info( "Adding previously dead node to list of active nodes" )
GlennRC632e2892015-10-19 18:58:41 -0700389 main.activeNodes.append( main.deadNode )
390
GlennRC632e2892015-10-19 18:58:41 -0700391 utilities.assert_equals( expect=main.TRUE,
392 actual=stepResult,
393 onpass="Successfully brought up onos node %s" % node,
394 onfail="Failed to bring up onos node %s" % node )
395
396
GlennRCe283c4b2016-01-07 13:04:10 -0800397 time.sleep(main.nodeSleep)
398
399 def CASE300( self, main ):
400 '''
401
402 Balancing Masters
403 '''
GlennRC632e2892015-10-19 18:58:41 -0700404
GlennRC475f50d2015-10-23 15:01:09 -0700405 main.step( "Balancing Masters" )
GlennRCe283c4b2016-01-07 13:04:10 -0800406
GlennRC475f50d2015-10-23 15:01:09 -0700407 stepResult = main.FALSE
408 if main.activeNodes:
409 controller = main.activeNodes[0]
YPZhang924ccfe2016-01-26 14:17:30 -0800410 stepResult = utilities.retry( main.CLIs[controller].balanceMasters,
411 main.FALSE,
412 [],
413 sleep=3,
414 attempts=3 )
415
GlennRCe283c4b2016-01-07 13:04:10 -0800416 else:
417 main.log.error( "List of active nodes is empty" )
GlennRC475f50d2015-10-23 15:01:09 -0700418 utilities.assert_equals( expect=main.TRUE,
419 actual=stepResult,
420 onpass="Balance masters was successfull",
421 onfail="Failed to balance masters")
GlennRC475f50d2015-10-23 15:01:09 -0700422 time.sleep(main.balanceSleep)
423
GlennRC632e2892015-10-19 18:58:41 -0700424 def CASE1000( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700425 '''
426 Report errors/warnings/exceptions
427 '''
GlennRC475f50d2015-10-23 15:01:09 -0700428 main.case( "Checking logs for errors, warnings, and exceptions" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700429 main.log.info("Error report: \n" )
430 main.ONOSbench.logReport( main.ONOSip[ 0 ],
GlennRC475f50d2015-10-23 15:01:09 -0700431 [ "INFO",
432 "FOLLOWER",
433 "WARN",
434 "flow",
435 "ERROR",
436 "Except" ],
437 "s" )