blob: 830fbf296beaf0c57a4b275aa11661fad9d994b6 [file] [log] [blame]
kelvin-onlab1d381fe2015-07-14 16:24:56 -07001
2# Testing network scalability, this test suite scales up a network topology
3# using mininet and verifies ONOS stability
4
GlennRC1c5df3c2015-08-27 16:12:09 -07005class SCPFscaleTopo:
kelvin-onlab1d381fe2015-07-14 16:24:56 -07006
7 def __init__( self ):
8 self.default = ''
9
10 def CASE1( self, main ):
11 import time
12 import os
13 import imp
Jon Hallf632d202015-07-30 15:45:11 -070014 import re
kelvin-onlab1d381fe2015-07-14 16:24:56 -070015
16 """
17 - Construct tests variables
18 - GIT ( optional )
19 - Checkout ONOS master branch
20 - Pull latest ONOS code
21 - Building ONOS ( optional )
22 - Install ONOS package
23 - Build ONOS package
24 """
25
GlennRC475f50d2015-10-23 15:01:09 -070026 main.case( "Constructing test variables" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070027 main.step( "Constructing test variables" )
28 stepResult = main.FALSE
29
GlennRC475f50d2015-10-23 15:01:09 -070030 main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
31 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
32 gitBranch = main.params[ 'GIT' ][ 'branch' ]
33 main.dependencyPath = main.testOnDirectory + \
34 main.params[ 'DEPENDENCY' ][ 'path' ]
35 main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
36 main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
37 main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] )
38 main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
39 main.topoScaleSize = len( main.topoScale )
40 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
41 wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
42 wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
43 main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] )
44 main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] )
45 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
GlennRC475f50d2015-10-23 15:01:09 -070046 main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
GlennRCe283c4b2016-01-07 13:04:10 -080047 main.nodeSleep = int( main.params[ 'SLEEP' ][ 'nodeSleep' ] )
GlennRC475f50d2015-10-23 15:01:09 -070048 main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] )
GlennRCe283c4b2016-01-07 13:04:10 -080049 main.MNSleep = int( main.params[ 'SLEEP' ][ 'MNsleep' ] )
YPZhang85024fc2016-02-09 16:59:27 -080050 main.pingTimeout = float( main.params[ 'TIMEOUT' ][ 'pingall' ] )
GlennRC475f50d2015-10-23 15:01:09 -070051 gitPull = main.params[ 'GIT' ][ 'pull' ]
52 main.homeDir = os.path.expanduser('~')
53 main.cellData = {} # for creating cell file
54 main.hostsData = {}
55 main.CLIs = []
56 main.ONOSip = []
57 main.activeNodes = []
58 main.ONOSip = main.ONOSbench.getOnosIps()
kelvin-onlab1d381fe2015-07-14 16:24:56 -070059
GlennRC475f50d2015-10-23 15:01:09 -070060 for i in range(main.numCtrls):
GlennRC632e2892015-10-19 18:58:41 -070061 main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
62
GlennRC475f50d2015-10-23 15:01:09 -070063 main.startUp = imp.load_source( wrapperFile1,
64 main.dependencyPath +
65 wrapperFile1 +
66 ".py" )
GlennRC632e2892015-10-19 18:58:41 -070067
GlennRC475f50d2015-10-23 15:01:09 -070068 main.scaleTopoFunction = imp.load_source( wrapperFile2,
69 main.dependencyPath +
70 wrapperFile2 +
71 ".py" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070072
GlennRC475f50d2015-10-23 15:01:09 -070073 main.topo = imp.load_source( wrapperFile3,
74 main.dependencyPath +
75 wrapperFile3 +
76 ".py" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070077
GlennRC632e2892015-10-19 18:58:41 -070078 main.ONOSbench.scp( main.Mininet1,
79 main.dependencyPath +
80 main.multiovs,
81 main.Mininet1.home,
82 direction="to" )
83
84 if main.CLIs:
85 stepResult = main.TRUE
86 else:
87 main.log.error( "Did not properly created list of " +
88 "ONOS CLI handle" )
89 stepResult = main.FALSE
90
kelvin-onlab1d381fe2015-07-14 16:24:56 -070091 utilities.assert_equals( expect=main.TRUE,
92 actual=stepResult,
93 onpass="Successfully construct " +
94 "test variables ",
95 onfail="Failed to construct test variables" )
96
97 if gitPull == 'True':
98 main.step( "Building ONOS in " + gitBranch + " branch" )
99 onosBuildResult = main.startUp.onosBuild( main, gitBranch )
100 stepResult = onosBuildResult
101 utilities.assert_equals( expect=main.TRUE,
102 actual=stepResult,
103 onpass="Successfully compiled " +
104 "latest ONOS",
105 onfail="Failed to compile " +
106 "latest ONOS" )
107 else:
108 main.log.warn( "Did not pull new code so skipping mvn " +
109 "clean install" )
110
GlennRC632e2892015-10-19 18:58:41 -0700111
112 def CASE2( self, main):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700113 """
114 - Set up cell
115 - Create cell file
116 - Set cell file
117 - Verify cell file
118 - Kill ONOS process
119 - Uninstall ONOS cluster
120 - Verify ONOS start up
121 - Install ONOS cluster
122 - Connect to cli
123 """
124
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700125 main.case( "Starting up " + str( main.numCtrls ) +
126 " node(s) ONOS cluster" )
GlennRC632e2892015-10-19 18:58:41 -0700127 main.caseExplanation = "Set up ONOS with " + str( main.numCtrls ) +\
128 " node(s) ONOS cluster"
129
130
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700131
132 #kill off all onos processes
133 main.log.info( "Safety check, killing all ONOS processes" +
Jon Hall70b2ff42015-11-17 15:49:44 -0800134 " before initiating environment setup" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700135
GlennRC632e2892015-10-19 18:58:41 -0700136 for i in range( main.numCtrls ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700137 main.ONOSbench.onosDie( main.ONOSip[ i ] )
138
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700139 tempOnosIp = []
140 for i in range( main.numCtrls ):
141 tempOnosIp.append( main.ONOSip[i] )
142
143 main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
144 "temp", main.Mininet1.ip_address,
GlennRC632e2892015-10-19 18:58:41 -0700145 main.apps, tempOnosIp )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700146
147 main.step( "Apply cell to environment" )
148 cellResult = main.ONOSbench.setCell( "temp" )
149 verifyResult = main.ONOSbench.verifyCell()
150 stepResult = cellResult and verifyResult
151 utilities.assert_equals( expect=main.TRUE,
152 actual=stepResult,
153 onpass="Successfully applied cell to " + \
154 "environment",
155 onfail="Failed to apply cell to environment " )
156
157 main.step( "Creating ONOS package" )
158 packageResult = main.ONOSbench.onosPackage()
159 stepResult = packageResult
160 utilities.assert_equals( expect=main.TRUE,
161 actual=stepResult,
162 onpass="Successfully created ONOS package",
163 onfail="Failed to create ONOS package" )
164
GlennRC632e2892015-10-19 18:58:41 -0700165 time.sleep( main.startUpSleep )
166 main.step( "Uninstalling ONOS package" )
167 onosUninstallResult = main.TRUE
168 for ip in main.ONOSip:
169 onosUninstallResult = onosUninstallResult and \
170 main.ONOSbench.onosUninstall( nodeIp=ip )
171 stepResult = onosUninstallResult
172 utilities.assert_equals( expect=main.TRUE,
173 actual=stepResult,
174 onpass="Successfully uninstalled ONOS package",
175 onfail="Failed to uninstall ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700176
GlennRC632e2892015-10-19 18:58:41 -0700177 time.sleep( main.startUpSleep )
178 main.step( "Installing ONOS package" )
179 onosInstallResult = main.TRUE
180 for i in range( main.numCtrls ):
181 onosInstallResult = onosInstallResult and \
182 main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
183 stepResult = onosInstallResult
184 utilities.assert_equals( expect=main.TRUE,
185 actual=stepResult,
186 onpass="Successfully installed ONOS package",
187 onfail="Failed to install ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700188
GlennRC632e2892015-10-19 18:58:41 -0700189 time.sleep( main.startUpSleep )
190 main.step( "Starting ONOS service" )
191 stopResult = main.TRUE
192 startResult = main.TRUE
193 onosIsUp = main.TRUE
194
195 for i in range( main.numCtrls ):
196 onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
197 if onosIsUp == main.TRUE:
198 main.log.report( "ONOS instance is up and ready" )
199 else:
200 main.log.report( "ONOS instance may not be up, stop and " +
201 "start ONOS again " )
202
203 for i in range( main.numCtrls ):
204 stopResult = stopResult and \
205 main.ONOSbench.onosStop( main.ONOSip[ i ] )
206 for i in range( main.numCtrls ):
207 startResult = startResult and \
208 main.ONOSbench.onosStart( main.ONOSip[ i ] )
209 stepResult = onosIsUp and stopResult and startResult
210 utilities.assert_equals( expect=main.TRUE,
211 actual=stepResult,
212 onpass="ONOS service is ready",
213 onfail="ONOS service did not start properly" )
214
215 main.step( "Start ONOS cli" )
216 cliResult = main.TRUE
217 for i in range( main.numCtrls ):
218 cliResult = cliResult and \
219 main.CLIs[ i ].startOnosCli( main.ONOSip[ i ] )
220 main.activeNodes.append( i )
221 stepResult = cliResult
222 utilities.assert_equals( expect=main.TRUE,
223 actual=stepResult,
224 onpass="Successfully start ONOS cli",
225 onfail="Failed to start ONOS cli" )
226
227
228 def CASE10( self, main ):
229 """
YPZhang85024fc2016-02-09 16:59:27 -0800230 Starting up torus topology
GlennRC632e2892015-10-19 18:58:41 -0700231 """
GlennRC475f50d2015-10-23 15:01:09 -0700232 import json
233
234 main.case( "Starting up Mininet and verifying topology" )
235 main.caseExplanation = "Starting Mininet with a scalling topology and " +\
236 "comparing topology elements between Mininet and ONOS"
GlennRC632e2892015-10-19 18:58:41 -0700237
238 main.log.info( "Checking if mininet is already running" )
239 if len( main.topoScale ) < main.topoScaleSize:
240 main.log.info( "Mininet is already running. Stopping mininet." )
241 main.Mininet1.stopNet()
GlennRCe283c4b2016-01-07 13:04:10 -0800242 time.sleep(main.MNSleep)
GlennRC632e2892015-10-19 18:58:41 -0700243 else:
244 main.log.info( "Mininet was not running" )
245
GlennRC475f50d2015-10-23 15:01:09 -0700246 if main.topoScale:
GlennRC90d43952015-10-27 11:36:15 -0700247 main.currScale = main.topoScale.pop(0)
GlennRC475f50d2015-10-23 15:01:09 -0700248 else: main.log.error( "topology scale is empty" )
GlennRC632e2892015-10-19 18:58:41 -0700249
GlennRC475f50d2015-10-23 15:01:09 -0700250
GlennRC90d43952015-10-27 11:36:15 -0700251 main.step( "Starting up TORUS %sx%s topology" % (main.currScale, main.currScale) )
GlennRC475f50d2015-10-23 15:01:09 -0700252
253 main.log.info( "Constructing Mininet command" )
254 mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs +\
GlennRC90d43952015-10-27 11:36:15 -0700255 " --switch ovsm --topo " + main.topoName + ","+ main.currScale + "," + main.currScale
GlennRC475f50d2015-10-23 15:01:09 -0700256
257 for i in range( main.numCtrls ):
258 mnCmd += " --controller remote,ip=" + main.ONOSip[ i ]
259
GlennRC632e2892015-10-19 18:58:41 -0700260 stepResult = main.Mininet1.startNet(mnCmd=mnCmd)
261 utilities.assert_equals( expect=main.TRUE,
262 actual=stepResult,
263 onpass=main.topoName +
264 " topology started successfully",
265 onfail=main.topoName +
266 " topology failed to start" )
267
GlennRCe283c4b2016-01-07 13:04:10 -0800268 time.sleep( main.MNSleep )
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700269
GlennRC475f50d2015-10-23 15:01:09 -0700270 def CASE11( self, main ):
271 """
272 Pingall, and compare topo
YPZhang85024fc2016-02-09 16:59:27 -0800273 We don't care the pingall result,
274 if the topology is same, then Pass.
GlennRC475f50d2015-10-23 15:01:09 -0700275 """
276 import json
277
GlennRC90d43952015-10-27 11:36:15 -0700278 main.case( "Verifying topology: TORUS %sx%s" % (main.currScale, main.currScale) )
YPZhang85024fc2016-02-09 16:59:27 -0800279 main.caseExplanation = "Pinging all hosts and comparing topology " +\
GlennRC475f50d2015-10-23 15:01:09 -0700280 "elements between Mininet and ONOS"
GlennRCe283c4b2016-01-07 13:04:10 -0800281
YPZhang85024fc2016-02-09 16:59:27 -0800282 main.log.info( "Pinging all hosts" )
283
284 # the pingall timeout is depend on the number of total host
285 pingTimeout = int( main.pingTimeout * float( int(main.currScale) * int(main.currScale ) ) )
286 pingResult = main.Mininet1.pingall( pingTimeout )
GlennRC475f50d2015-10-23 15:01:09 -0700287
288 main.log.info( "Gathering topology information" )
YPZhang85024fc2016-02-09 16:59:27 -0800289
290 time.sleep( main.MNSleep )
291
GlennRC475f50d2015-10-23 15:01:09 -0700292 devicesResults = main.TRUE
293 linksResults = main.TRUE
294 hostsResults = main.TRUE
YPZhang85024fc2016-02-09 16:59:27 -0800295 stepResult = main.TRUE
GlennRC475f50d2015-10-23 15:01:09 -0700296 devices = main.topo.getAllDevices( main )
297 hosts = main.topo.getAllHosts( main )
298 ports = main.topo.getAllPorts( main )
299 links = main.topo.getAllLinks( main )
300 clusters = main.topo.getAllClusters( main )
GlennRC475f50d2015-10-23 15:01:09 -0700301 mnSwitches = main.Mininet1.getSwitches()
302 mnLinks = main.Mininet1.getLinks()
303 mnHosts = main.Mininet1.getHosts()
304
305 main.step( "Comparing MN topology to ONOS topology" )
GlennRC475f50d2015-10-23 15:01:09 -0700306 for controller in range(len(main.activeNodes)):
307 controllerStr = str( main.activeNodes[controller] + 1 )
308 if devices[ controller ] and ports[ controller ] and\
309 "Error" not in devices[ controller ] and\
310 "Error" not in ports[ controller ]:
311
312 currentDevicesResult = main.Mininet1.compareSwitches(
313 mnSwitches,
314 json.loads( devices[ controller ] ),
315 json.loads( ports[ controller ] ) )
316 else:
317 currentDevicesResult = main.FALSE
YPZhang85024fc2016-02-09 16:59:27 -0800318
GlennRC475f50d2015-10-23 15:01:09 -0700319 if links[ controller ] and "Error" not in links[ controller ]:
320 currentLinksResult = main.Mininet1.compareLinks(
321 mnSwitches, mnLinks,
322 json.loads( links[ controller ] ) )
323 else:
324 currentLinksResult = main.FALSE
GlennRC475f50d2015-10-23 15:01:09 -0700325
326 if hosts[ controller ] or "Error" not in hosts[ controller ]:
327 currentHostsResult = main.Mininet1.compareHosts(
328 mnHosts,
329 json.loads( hosts[ controller ] ) )
330 else:
331 currentHostsResult = main.FALSE
YPZhang85024fc2016-02-09 16:59:27 -0800332
333 stepResult = stepResult and currentDevicesResult and currentLinksResult and currentHostsResult
334
GlennRC475f50d2015-10-23 15:01:09 -0700335 utilities.assert_equals( expect=main.TRUE,
YPZhang85024fc2016-02-09 16:59:27 -0800336 actual=stepResult,
337 onpass=" Topology match Mininet",
GlennRC475f50d2015-10-23 15:01:09 -0700338 onfail="ONOS" + controllerStr +
YPZhang85024fc2016-02-09 16:59:27 -0800339 " Topology doesn't match Mininet" )
GlennRC475f50d2015-10-23 15:01:09 -0700340
341
GlennRC632e2892015-10-19 18:58:41 -0700342 def CASE100( self, main ):
343 '''
YPZhang77badfc2016-03-09 10:28:59 -0800344 Bring Down node 3
GlennRC632e2892015-10-19 18:58:41 -0700345 '''
GlennRC475f50d2015-10-23 15:01:09 -0700346
GlennRC90d43952015-10-27 11:36:15 -0700347 main.case("Balancing Masters and bring ONOS node 3 down: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700348 main.caseExplanation = "Balance masters to make sure " +\
349 "each controller has some devices and " +\
350 "stop ONOS node 3 service. "
351
GlennRC475f50d2015-10-23 15:01:09 -0700352 stepResult = main.FALSE
GlennRCed2122e2015-10-21 14:38:46 -0700353 main.step( "Bringing down node 3" )
GlennRCed2122e2015-10-21 14:38:46 -0700354 # Always bring down the third node
355 main.deadNode = 2
GlennRCed2122e2015-10-21 14:38:46 -0700356 # Printing purposes
GlennRC632e2892015-10-19 18:58:41 -0700357 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700358 main.log.info( "Stopping node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700359 stepResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] )
GlennRC475f50d2015-10-23 15:01:09 -0700360 main.log.info( "Removing dead node from list of active nodes" )
361 main.activeNodes.pop( main.deadNode )
GlennRC632e2892015-10-19 18:58:41 -0700362
YPZhang77badfc2016-03-09 10:28:59 -0800363 utilities.assert_equals( expect=main.TRUE,
364 actual=stepResult,
365 onpass="Successfully bring down node 3",
366 onfail="Failed to bring down node 3" )
GlennRC632e2892015-10-19 18:58:41 -0700367
GlennRC475f50d2015-10-23 15:01:09 -0700368 def CASE200( self, main ):
GlennRC632e2892015-10-19 18:58:41 -0700369 '''
GlennRC475f50d2015-10-23 15:01:09 -0700370 Bring up onos node and balance masters
GlennRC632e2892015-10-19 18:58:41 -0700371 '''
GlennRC475f50d2015-10-23 15:01:09 -0700372
GlennRC90d43952015-10-27 11:36:15 -0700373 main.case("Bring ONOS node 3 up and balance masters: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700374 main.caseExplanation = "Bring node 3 back up and balance the masters"
GlennRC632e2892015-10-19 18:58:41 -0700375
376 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700377 main.log.info( "Starting node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700378 stepResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700379 main.log.info( "Starting onos cli" )
GlennRC475f50d2015-10-23 15:01:09 -0700380 stepResult = stepResult and main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700381
GlennRC475f50d2015-10-23 15:01:09 -0700382 main.log.info( "Adding previously dead node to list of active nodes" )
GlennRC632e2892015-10-19 18:58:41 -0700383 main.activeNodes.append( main.deadNode )
384
GlennRC632e2892015-10-19 18:58:41 -0700385 utilities.assert_equals( expect=main.TRUE,
386 actual=stepResult,
387 onpass="Successfully brought up onos node %s" % node,
388 onfail="Failed to bring up onos node %s" % node )
389
390
GlennRCe283c4b2016-01-07 13:04:10 -0800391 time.sleep(main.nodeSleep)
392
393 def CASE300( self, main ):
394 '''
395
396 Balancing Masters
397 '''
YPZhang85024fc2016-02-09 16:59:27 -0800398 time.sleep(main.balanceSleep)
GlennRC475f50d2015-10-23 15:01:09 -0700399 main.step( "Balancing Masters" )
GlennRCe283c4b2016-01-07 13:04:10 -0800400
GlennRC475f50d2015-10-23 15:01:09 -0700401 stepResult = main.FALSE
402 if main.activeNodes:
403 controller = main.activeNodes[0]
YPZhang924ccfe2016-01-26 14:17:30 -0800404 stepResult = utilities.retry( main.CLIs[controller].balanceMasters,
405 main.FALSE,
406 [],
407 sleep=3,
408 attempts=3 )
409
GlennRCe283c4b2016-01-07 13:04:10 -0800410 else:
411 main.log.error( "List of active nodes is empty" )
GlennRC475f50d2015-10-23 15:01:09 -0700412 utilities.assert_equals( expect=main.TRUE,
413 actual=stepResult,
414 onpass="Balance masters was successfull",
415 onfail="Failed to balance masters")
GlennRC475f50d2015-10-23 15:01:09 -0700416 time.sleep(main.balanceSleep)
417
GlennRC632e2892015-10-19 18:58:41 -0700418 def CASE1000( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700419 '''
420 Report errors/warnings/exceptions
421 '''
GlennRC475f50d2015-10-23 15:01:09 -0700422 main.case( "Checking logs for errors, warnings, and exceptions" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700423 main.log.info("Error report: \n" )
424 main.ONOSbench.logReport( main.ONOSip[ 0 ],
GlennRC475f50d2015-10-23 15:01:09 -0700425 [ "INFO",
426 "FOLLOWER",
427 "WARN",
428 "flow",
429 "ERROR",
430 "Except" ],
431 "s" )