blob: 815d772b810489556a3d76fa5768dc7346db8c36 [file] [log] [blame]
kelvin-onlab1d381fe2015-07-14 16:24:56 -07001
2# Testing network scalability, this test suite scales up a network topology
3# using mininet and verifies ONOS stability
4
GlennRC1c5df3c2015-08-27 16:12:09 -07005class SCPFscaleTopo:
kelvin-onlab1d381fe2015-07-14 16:24:56 -07006
7 def __init__( self ):
8 self.default = ''
9
10 def CASE1( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -070011 import os
12 import imp
13
14 """
15 - Construct tests variables
16 - GIT ( optional )
17 - Checkout ONOS master branch
18 - Pull latest ONOS code
19 - Building ONOS ( optional )
20 - Install ONOS package
21 - Build ONOS package
22 """
GlennRC475f50d2015-10-23 15:01:09 -070023 main.case( "Constructing test variables" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -070024 main.step( "Constructing test variables" )
25 stepResult = main.FALSE
Chiyu Chengb8c2c842016-10-05 12:40:49 -070026 # The variable to decide if the data should be written into data base.
27 # 1 means Yes and -1 means No.
28 main.writeData = 1
29 main.searchTerm = main.params[ 'SearchTerm' ]
GlennRC475f50d2015-10-23 15:01:09 -070030 main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
31 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
32 gitBranch = main.params[ 'GIT' ][ 'branch' ]
33 main.dependencyPath = main.testOnDirectory + \
34 main.params[ 'DEPENDENCY' ][ 'path' ]
35 main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
36 main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
37 main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] )
38 main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
39 main.topoScaleSize = len( main.topoScale )
40 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
41 wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
42 wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
43 main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] )
44 main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] )
45 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
GlennRC475f50d2015-10-23 15:01:09 -070046 main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
GlennRCe283c4b2016-01-07 13:04:10 -080047 main.nodeSleep = int( main.params[ 'SLEEP' ][ 'nodeSleep' ] )
GlennRC475f50d2015-10-23 15:01:09 -070048 main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] )
GlennRCe283c4b2016-01-07 13:04:10 -080049 main.MNSleep = int( main.params[ 'SLEEP' ][ 'MNsleep' ] )
YPZhang85024fc2016-02-09 16:59:27 -080050 main.pingTimeout = float( main.params[ 'TIMEOUT' ][ 'pingall' ] )
YPZhangacaaf422016-07-26 09:34:03 -070051 main.hostDiscover = main.params[ 'TOPOLOGY' ][ 'host' ]
52 main.hostDiscoverSleep = float( main.params['SLEEP']['host'] )
53 if main.hostDiscover == 'True':
54 main.hostDiscover = True
55 else:
56 main.hostDiscover = False
GlennRC475f50d2015-10-23 15:01:09 -070057 gitPull = main.params[ 'GIT' ][ 'pull' ]
58 main.homeDir = os.path.expanduser('~')
59 main.cellData = {} # for creating cell file
60 main.hostsData = {}
61 main.CLIs = []
62 main.ONOSip = []
63 main.activeNodes = []
64 main.ONOSip = main.ONOSbench.getOnosIps()
kelvin-onlab1d381fe2015-07-14 16:24:56 -070065
GlennRC475f50d2015-10-23 15:01:09 -070066 for i in range(main.numCtrls):
GlennRC632e2892015-10-19 18:58:41 -070067 main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
68
Chiyu Chengb8c2c842016-10-05 12:40:49 -070069 main.allinfo = {} # The dictionary to record all the data from karaf.log
70 for i in range( 2 ):
71 main.allinfo[ i ]={}
72 for w in range ( 3 ):
73 # Totaltime: the time from the new switchConnection to its end
74 # swConnection: the time from the first new switchConnection to the last new switchConnection
75 # disconnectRate: the rate that shows how many switch disconnect after connection
Chiyu Cheng8800c7c2016-10-18 11:26:35 -070076 main.allinfo[ i ][ 'info' + str( w ) ]= { 'totalTime': 0, 'swConnection': 0,'lastSwToTopology': 0, 'disconnectRate': 0 }
Chiyu Chengb8c2c842016-10-05 12:40:49 -070077
78 main.dbFilePath = main.params[ 'DATABASE' ][ 'dbPath' ]
79 main.log.info( "Create Database file " + main.dbFilePath )
80 resultDB = open(main.dbFilePath, 'w+' )
81 resultDB.close()
82
GlennRC475f50d2015-10-23 15:01:09 -070083 main.startUp = imp.load_source( wrapperFile1,
84 main.dependencyPath +
85 wrapperFile1 +
86 ".py" )
GlennRC475f50d2015-10-23 15:01:09 -070087 main.scaleTopoFunction = imp.load_source( wrapperFile2,
88 main.dependencyPath +
89 wrapperFile2 +
90 ".py" )
GlennRC475f50d2015-10-23 15:01:09 -070091 main.topo = imp.load_source( wrapperFile3,
92 main.dependencyPath +
93 wrapperFile3 +
94 ".py" )
GlennRC632e2892015-10-19 18:58:41 -070095 main.ONOSbench.scp( main.Mininet1,
96 main.dependencyPath +
97 main.multiovs,
98 main.Mininet1.home,
99 direction="to" )
100
101 if main.CLIs:
102 stepResult = main.TRUE
103 else:
104 main.log.error( "Did not properly created list of " +
105 "ONOS CLI handle" )
106 stepResult = main.FALSE
107
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700108 utilities.assert_equals( expect=main.TRUE,
109 actual=stepResult,
110 onpass="Successfully construct " +
111 "test variables ",
112 onfail="Failed to construct test variables" )
113
114 if gitPull == 'True':
115 main.step( "Building ONOS in " + gitBranch + " branch" )
116 onosBuildResult = main.startUp.onosBuild( main, gitBranch )
117 stepResult = onosBuildResult
118 utilities.assert_equals( expect=main.TRUE,
119 actual=stepResult,
120 onpass="Successfully compiled " +
121 "latest ONOS",
122 onfail="Failed to compile " +
123 "latest ONOS" )
124 else:
125 main.log.warn( "Did not pull new code so skipping mvn " +
126 "clean install" )
127
GlennRC632e2892015-10-19 18:58:41 -0700128
129 def CASE2( self, main):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700130 """
131 - Set up cell
132 - Create cell file
133 - Set cell file
134 - Verify cell file
135 - Kill ONOS process
136 - Uninstall ONOS cluster
137 - Verify ONOS start up
138 - Install ONOS cluster
139 - Connect to cli
140 """
YPZhangacaaf422016-07-26 09:34:03 -0700141 import time
YPZhang29c2d642016-06-22 16:15:19 -0700142 main.log.info( "Checking if mininet is already running" )
143 if len( main.topoScale ) < main.topoScaleSize:
144 main.log.info( "Mininet is already running. Stopping mininet." )
145 main.Mininet1.stopNet()
146 time.sleep(main.MNSleep)
147 else:
148 main.log.info( "Mininet was not running" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700149
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700150 main.case( "Starting up " + str( main.numCtrls ) +
151 " node(s) ONOS cluster" )
GlennRC632e2892015-10-19 18:58:41 -0700152 main.caseExplanation = "Set up ONOS with " + str( main.numCtrls ) +\
153 " node(s) ONOS cluster"
154
155
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700156
157 #kill off all onos processes
158 main.log.info( "Safety check, killing all ONOS processes" +
Jon Hall70b2ff42015-11-17 15:49:44 -0800159 " before initiating environment setup" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700160
GlennRC632e2892015-10-19 18:58:41 -0700161 for i in range( main.numCtrls ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700162 main.ONOSbench.onosDie( main.ONOSip[ i ] )
163
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700164 tempOnosIp = []
165 for i in range( main.numCtrls ):
166 tempOnosIp.append( main.ONOSip[i] )
167
168 main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
169 "temp", main.Mininet1.ip_address,
GlennRC632e2892015-10-19 18:58:41 -0700170 main.apps, tempOnosIp )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700171
172 main.step( "Apply cell to environment" )
173 cellResult = main.ONOSbench.setCell( "temp" )
174 verifyResult = main.ONOSbench.verifyCell()
175 stepResult = cellResult and verifyResult
176 utilities.assert_equals( expect=main.TRUE,
177 actual=stepResult,
178 onpass="Successfully applied cell to " + \
179 "environment",
180 onfail="Failed to apply cell to environment " )
181
182 main.step( "Creating ONOS package" )
Jon Hallbd60ea02016-08-23 10:03:59 -0700183 packageResult = main.ONOSbench.buckBuild()
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700184 stepResult = packageResult
185 utilities.assert_equals( expect=main.TRUE,
186 actual=stepResult,
187 onpass="Successfully created ONOS package",
188 onfail="Failed to create ONOS package" )
189
GlennRC632e2892015-10-19 18:58:41 -0700190 time.sleep( main.startUpSleep )
191 main.step( "Uninstalling ONOS package" )
192 onosUninstallResult = main.TRUE
193 for ip in main.ONOSip:
194 onosUninstallResult = onosUninstallResult and \
195 main.ONOSbench.onosUninstall( nodeIp=ip )
196 stepResult = onosUninstallResult
197 utilities.assert_equals( expect=main.TRUE,
198 actual=stepResult,
199 onpass="Successfully uninstalled ONOS package",
200 onfail="Failed to uninstall ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700201
GlennRC632e2892015-10-19 18:58:41 -0700202 time.sleep( main.startUpSleep )
203 main.step( "Installing ONOS package" )
204 onosInstallResult = main.TRUE
205 for i in range( main.numCtrls ):
206 onosInstallResult = onosInstallResult and \
207 main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
208 stepResult = onosInstallResult
209 utilities.assert_equals( expect=main.TRUE,
210 actual=stepResult,
211 onpass="Successfully installed ONOS package",
212 onfail="Failed to install ONOS package" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700213
GlennRC632e2892015-10-19 18:58:41 -0700214 time.sleep( main.startUpSleep )
215 main.step( "Starting ONOS service" )
216 stopResult = main.TRUE
217 startResult = main.TRUE
218 onosIsUp = main.TRUE
219
220 for i in range( main.numCtrls ):
221 onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
222 if onosIsUp == main.TRUE:
223 main.log.report( "ONOS instance is up and ready" )
224 else:
225 main.log.report( "ONOS instance may not be up, stop and " +
226 "start ONOS again " )
227
228 for i in range( main.numCtrls ):
229 stopResult = stopResult and \
230 main.ONOSbench.onosStop( main.ONOSip[ i ] )
231 for i in range( main.numCtrls ):
232 startResult = startResult and \
233 main.ONOSbench.onosStart( main.ONOSip[ i ] )
234 stepResult = onosIsUp and stopResult and startResult
235 utilities.assert_equals( expect=main.TRUE,
236 actual=stepResult,
237 onpass="ONOS service is ready",
238 onfail="ONOS service did not start properly" )
239
240 main.step( "Start ONOS cli" )
241 cliResult = main.TRUE
YPZhang29c2d642016-06-22 16:15:19 -0700242 main.activeNodes = []
GlennRC632e2892015-10-19 18:58:41 -0700243 for i in range( main.numCtrls ):
244 cliResult = cliResult and \
245 main.CLIs[ i ].startOnosCli( main.ONOSip[ i ] )
246 main.activeNodes.append( i )
247 stepResult = cliResult
248 utilities.assert_equals( expect=main.TRUE,
249 actual=stepResult,
250 onpass="Successfully start ONOS cli",
251 onfail="Failed to start ONOS cli" )
YPZhang29c2d642016-06-22 16:15:19 -0700252 time.sleep( main.startUpSleep )
GlennRC632e2892015-10-19 18:58:41 -0700253
254 def CASE10( self, main ):
255 """
YPZhang85024fc2016-02-09 16:59:27 -0800256 Starting up torus topology
GlennRC632e2892015-10-19 18:58:41 -0700257 """
GlennRC475f50d2015-10-23 15:01:09 -0700258
259 main.case( "Starting up Mininet and verifying topology" )
260 main.caseExplanation = "Starting Mininet with a scalling topology and " +\
261 "comparing topology elements between Mininet and ONOS"
GlennRC475f50d2015-10-23 15:01:09 -0700262 if main.topoScale:
GlennRC90d43952015-10-27 11:36:15 -0700263 main.currScale = main.topoScale.pop(0)
GlennRC475f50d2015-10-23 15:01:09 -0700264 else: main.log.error( "topology scale is empty" )
GlennRC90d43952015-10-27 11:36:15 -0700265 main.step( "Starting up TORUS %sx%s topology" % (main.currScale, main.currScale) )
GlennRC475f50d2015-10-23 15:01:09 -0700266
267 main.log.info( "Constructing Mininet command" )
YPZhangacaaf422016-07-26 09:34:03 -0700268 mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs + \
269 " --switch ovsm --topo " + main.topoName + "," + main.currScale + "," + main.currScale
GlennRC475f50d2015-10-23 15:01:09 -0700270 for i in range( main.numCtrls ):
271 mnCmd += " --controller remote,ip=" + main.ONOSip[ i ]
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700272 stepResult = main.Mininet1.startNet( mnCmd=mnCmd )
GlennRC632e2892015-10-19 18:58:41 -0700273 utilities.assert_equals( expect=main.TRUE,
274 actual=stepResult,
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700275 onpass=main.topoName +
GlennRC632e2892015-10-19 18:58:41 -0700276 " topology started successfully",
277 onfail=main.topoName +
278 " topology failed to start" )
279
GlennRCe283c4b2016-01-07 13:04:10 -0800280 time.sleep( main.MNSleep )
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700281
GlennRC475f50d2015-10-23 15:01:09 -0700282 def CASE11( self, main ):
283 """
YPZhangacaaf422016-07-26 09:34:03 -0700284 Compare topo, and sending Arping package
YPZhang85024fc2016-02-09 16:59:27 -0800285 if the topology is same, then Pass.
GlennRC475f50d2015-10-23 15:01:09 -0700286 """
287 import json
YPZhangacaaf422016-07-26 09:34:03 -0700288 import time
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700289 # First capture
290 for i in range( 3 ):
291 # Calculate total time
292 main.allinfo[ 0 ][ 'info' + str( i )][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'end' ], 'last', index=i, funcMode='TD' )
293 # Calculate switch connection time
294 main.allinfo[ 0 ][ 'info' + str( i )][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'start' ], 'last', index=i, funcMode='TD' )
Chiyu Cheng8800c7c2016-10-18 11:26:35 -0700295 # Calculate the time from last switch connection to the end
296 main.allinfo[ 0 ][ 'info' + str( i )][ 'lastSwToTopology' ] = main.allinfo[ 0 ][ 'info' + str( i )][ 'totalTime' ] - main.allinfo[ 0 ][ 'info' + str( i )][ 'swConnection' ]
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700297 # Calculate the disconnecti rate
298 main.allinfo[ 0 ][ 'info' + str( i )][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'Disconnect' ], 'num', main.searchTerm[ 'start' ], 'num', index=i, funcMode='DR' )
299 main.log.debug( "The data is " + str( main.allinfo[ 0 ] ) )
GlennRC475f50d2015-10-23 15:01:09 -0700300
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700301 main.case( "Verifying topology: TORUS %sx%s" % ( main.currScale, main.currScale ) )
YPZhang85024fc2016-02-09 16:59:27 -0800302 main.caseExplanation = "Pinging all hosts and comparing topology " +\
GlennRC475f50d2015-10-23 15:01:09 -0700303 "elements between Mininet and ONOS"
GlennRCe283c4b2016-01-07 13:04:10 -0800304
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700305 main.log.info( "Gathering topology information")
YPZhang85024fc2016-02-09 16:59:27 -0800306 time.sleep( main.MNSleep )
YPZhang85024fc2016-02-09 16:59:27 -0800307 stepResult = main.TRUE
GlennRC475f50d2015-10-23 15:01:09 -0700308 main.step( "Comparing MN topology to ONOS topology" )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700309 compareRetry = 0
310 while compareRetry < 3:
YPZhang81a7d4e2016-04-18 13:10:17 -0700311 #While loop for retry
312 devices = main.topo.getAllDevices( main )
YPZhang81a7d4e2016-04-18 13:10:17 -0700313 ports = main.topo.getAllPorts( main )
314 links = main.topo.getAllLinks( main)
YPZhang81a7d4e2016-04-18 13:10:17 -0700315 mnSwitches = main.Mininet1.getSwitches()
316 mnLinks = main.Mininet1.getLinks(timeout=180)
GlennRC475f50d2015-10-23 15:01:09 -0700317
YPZhang81a7d4e2016-04-18 13:10:17 -0700318 for controller in range(len(main.activeNodes)):
YPZhangacaaf422016-07-26 09:34:03 -0700319 # controllerStr = str( main.activeNodes[controller] + 1 )
320 if devices[ controller ] and ports[ controller ] and \
321 "Error" not in devices[ controller ] and \
322 "Error" not in ports[ controller ]:
YPZhang81a7d4e2016-04-18 13:10:17 -0700323 currentDevicesResult = main.Mininet1.compareSwitches(
324 mnSwitches,
325 json.loads( devices[ controller ] ),
326 json.loads( ports[ controller ] ) )
327 else:
328 currentDevicesResult = main.FALSE
329
330 if links[ controller ] and "Error" not in links[ controller ]:
331 currentLinksResult = main.Mininet1.compareLinks(
332 mnSwitches, mnLinks,
333 json.loads( links[ controller ] ) )
334 else:
335 currentLinksResult = main.FALSE
336
YPZhangacaaf422016-07-26 09:34:03 -0700337 stepResult = stepResult and currentDevicesResult and currentLinksResult
YPZhang81a7d4e2016-04-18 13:10:17 -0700338 if stepResult:
339 break
340 compareRetry += 1
YPZhangacaaf422016-07-26 09:34:03 -0700341 utilities.assert_equals(expect=main.TRUE,
342 actual=stepResult,
343 onpass=" Topology match Mininet",
344 onfail="ONOS Topology doesn't match Mininet")
YPZhang81a7d4e2016-04-18 13:10:17 -0700345
YPZhangacaaf422016-07-26 09:34:03 -0700346 if stepResult:
347 if main.hostDiscover:
348 hostList = []
349 for i in range( 1, int( main.currScale ) + 1 ):
350 for j in range( 1, int( main.currScale ) + 1) :
351 # Generate host list
352 hoststr = "h" + str(i) + "x" + str(j)
353 hostList.append(hoststr)
354 for i in range( len(hostList) ):
355 totalHost = main.topo.sendArpPackage( main, hostList[i] )
356 time.sleep( main.hostDiscoverSleep )
357 if totalHost < 0:
358 # if totalHost less than 0 which means dependence function has exception.
359 main.log.info( "Error when discover host!" )
360 break
361 if totalHost == int( main.currScale ) * int( main.currScale ):
362 main.log.info( "Discovered all hosts" )
You Wang7bb9c462016-08-10 14:18:16 -0700363 stepResult = stepResult and main.TRUE
YPZhangacaaf422016-07-26 09:34:03 -0700364 else:
365 main.log.warn( "Some hosts ware not discovered by ONOS... Topology doesn't match!" )
You Wang7bb9c462016-08-10 14:18:16 -0700366 stepResult = main.FALSE
YPZhangacaaf422016-07-26 09:34:03 -0700367 utilities.assert_equals(expect=main.TRUE,
368 actual=stepResult,
369 onpass=" Topology match Mininet",
370 onfail="ONOS Topology doesn't match Mininet")
371 main.log.info( "Finished this iteration, continue to scale next topology." )
YPZhang81a7d4e2016-04-18 13:10:17 -0700372 else:
YPZhangacaaf422016-07-26 09:34:03 -0700373 main.log.info( "Clean up and exit TestON. Finished this test." )
374 main.cleanup()
375 main.exit()
GlennRC475f50d2015-10-23 15:01:09 -0700376
GlennRC632e2892015-10-19 18:58:41 -0700377 def CASE100( self, main ):
378 '''
YPZhang81a7d4e2016-04-18 13:10:17 -0700379 Bring Down node 3
GlennRC632e2892015-10-19 18:58:41 -0700380 '''
GlennRC475f50d2015-10-23 15:01:09 -0700381
YPZhangacaaf422016-07-26 09:34:03 -0700382 main.case("Bring ONOS node 3 down: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700383 main.caseExplanation = "Balance masters to make sure " +\
384 "each controller has some devices and " +\
385 "stop ONOS node 3 service. "
386
GlennRC475f50d2015-10-23 15:01:09 -0700387 stepResult = main.FALSE
GlennRCed2122e2015-10-21 14:38:46 -0700388 main.step( "Bringing down node 3" )
GlennRCed2122e2015-10-21 14:38:46 -0700389 # Always bring down the third node
390 main.deadNode = 2
GlennRCed2122e2015-10-21 14:38:46 -0700391 # Printing purposes
GlennRC632e2892015-10-19 18:58:41 -0700392 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700393 main.log.info( "Stopping node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700394 stepResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] )
GlennRC475f50d2015-10-23 15:01:09 -0700395 main.log.info( "Removing dead node from list of active nodes" )
396 main.activeNodes.pop( main.deadNode )
GlennRC632e2892015-10-19 18:58:41 -0700397
YPZhang77badfc2016-03-09 10:28:59 -0800398 utilities.assert_equals( expect=main.TRUE,
399 actual=stepResult,
400 onpass="Successfully bring down node 3",
401 onfail="Failed to bring down node 3" )
GlennRC632e2892015-10-19 18:58:41 -0700402
GlennRC475f50d2015-10-23 15:01:09 -0700403 def CASE200( self, main ):
GlennRC632e2892015-10-19 18:58:41 -0700404 '''
YPZhangacaaf422016-07-26 09:34:03 -0700405 Bring up onos node
GlennRC632e2892015-10-19 18:58:41 -0700406 '''
GlennRC475f50d2015-10-23 15:01:09 -0700407
YPZhangacaaf422016-07-26 09:34:03 -0700408 main.case("Bring ONOS node 3 up: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700409 main.caseExplanation = "Bring node 3 back up and balance the masters"
GlennRC632e2892015-10-19 18:58:41 -0700410
411 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700412 main.log.info( "Starting node %s" % node )
GlennRC475f50d2015-10-23 15:01:09 -0700413 stepResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700414 main.log.info( "Starting onos cli" )
GlennRC475f50d2015-10-23 15:01:09 -0700415 stepResult = stepResult and main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] )
GlennRC632e2892015-10-19 18:58:41 -0700416
GlennRC475f50d2015-10-23 15:01:09 -0700417 main.log.info( "Adding previously dead node to list of active nodes" )
GlennRC632e2892015-10-19 18:58:41 -0700418 main.activeNodes.append( main.deadNode )
419
GlennRC632e2892015-10-19 18:58:41 -0700420 utilities.assert_equals( expect=main.TRUE,
421 actual=stepResult,
422 onpass="Successfully brought up onos node %s" % node,
423 onfail="Failed to bring up onos node %s" % node )
424
425
GlennRCe283c4b2016-01-07 13:04:10 -0800426 time.sleep(main.nodeSleep)
427
428 def CASE300( self, main ):
429 '''
430
431 Balancing Masters
432 '''
YPZhang85024fc2016-02-09 16:59:27 -0800433 time.sleep(main.balanceSleep)
GlennRC475f50d2015-10-23 15:01:09 -0700434 main.step( "Balancing Masters" )
GlennRCe283c4b2016-01-07 13:04:10 -0800435
GlennRC475f50d2015-10-23 15:01:09 -0700436 stepResult = main.FALSE
437 if main.activeNodes:
438 controller = main.activeNodes[0]
YPZhang924ccfe2016-01-26 14:17:30 -0800439 stepResult = utilities.retry( main.CLIs[controller].balanceMasters,
440 main.FALSE,
441 [],
442 sleep=3,
443 attempts=3 )
444
GlennRCe283c4b2016-01-07 13:04:10 -0800445 else:
446 main.log.error( "List of active nodes is empty" )
GlennRC475f50d2015-10-23 15:01:09 -0700447 utilities.assert_equals( expect=main.TRUE,
448 actual=stepResult,
449 onpass="Balance masters was successfull",
450 onfail="Failed to balance masters")
GlennRC475f50d2015-10-23 15:01:09 -0700451 time.sleep(main.balanceSleep)
452
GlennRC632e2892015-10-19 18:58:41 -0700453 def CASE1000( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700454 '''
455 Report errors/warnings/exceptions
456 '''
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700457 # Compare the slowest Node through total time of each node
458 slowestNode = 0
459 slowestTotalTime = 0
460 # Second capture
461 for i in range( 3 ):
462 # Calculate total time
463 main.allinfo[ 1 ][ 'info' + str( i )][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'end' ], 'last', index=i, funcMode='TD' )
464 # Compare the total time
465 if main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ] > slowestTotalTime:
466 slowestTotalTime = main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ]
467 slowestNode = i
468 # Calculate switch connection time
469 main.allinfo[ 1 ][ 'info' + str( i )][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'start' ], 'last', index=i, funcMode='TD' )
Chiyu Cheng8800c7c2016-10-18 11:26:35 -0700470 # Calculate the time from last switch connection to the end
471 main.allinfo[ 1 ][ 'info' + str( i )][ 'lastSwToTopology' ] = main.allinfo[ 1 ][
472 'info' + str( i )][ 'totalTime' ] - main.allinfo[ 1 ][ 'info' + str( i )][
473 'swConnection' ]
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700474 # Calculate the disconnecti rate
475 main.allinfo[ 1 ][ 'info' + str( i )][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'Disconnect' ], 'num', main.searchTerm[ 'start' ],'num', index=i, funcMode='DR' )
476
477 if ( main.allinfo[ 0 ] != main.allinfo[ 1 ] ):
478 main.log.error( "The results of two capture are different!" )
479 main.log.debug( "The data is " + str( main.allinfo ) )
480 if main.writeData != -1:
481 main.log.info( "Write the date into database" )
482 # write the date into data base
483 with open( main.dbFilePath, "a" ) as dbFile:
484 temp = str( main.currScale )
485 temp += ",'baremetal1'"
486 # put result from second capture into data base
487 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'totalTime' ] )
488 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'swConnection' ] )
Chiyu Cheng8800c7c2016-10-18 11:26:35 -0700489 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'lastSwToTopology' ] )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700490 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'disconnectRate' ] )
491 temp += "\n"
492 dbFile.write( temp )
493 else:
494 main.log.error( "The data from log is wrong!" )
495 main.writeData = 1
GlennRC475f50d2015-10-23 15:01:09 -0700496 main.case( "Checking logs for errors, warnings, and exceptions" )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700497 main.log.info( "Error report: \n" )
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700498 main.ONOSbench.logReport( main.ONOSip[ 0 ],
GlennRC475f50d2015-10-23 15:01:09 -0700499 [ "INFO",
500 "FOLLOWER",
501 "WARN",
502 "flow",
503 "ERROR",
504 "Except" ],
505 "s" )