blob: b0b888047303b3a0c2d42563084a7ed858cdb62d [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2015 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
kelvin-onlab1d381fe2015-07-14 16:24:56 -070021# Testing network scalability, this test suite scales up a network topology
22# using mininet and verifies ONOS stability
23
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070024
GlennRC1c5df3c2015-08-27 16:12:09 -070025class SCPFscaleTopo:
kelvin-onlab1d381fe2015-07-14 16:24:56 -070026
27 def __init__( self ):
28 self.default = ''
29
30 def CASE1( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -070031 import os
32 import imp
kelvin-onlab1d381fe2015-07-14 16:24:56 -070033 """
34 - Construct tests variables
35 - GIT ( optional )
36 - Checkout ONOS master branch
37 - Pull latest ONOS code
38 - Building ONOS ( optional )
39 - Install ONOS package
40 - Build ONOS package
41 """
Devin Lim58046fa2017-07-05 16:55:00 -070042 try:
43 from tests.dependencies.ONOSSetup import ONOSSetup
44 main.testSetUp = ONOSSetup()
45 except ImportError:
46 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070047 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070048 main.testSetUp.envSetupDescription()
kelvin-onlab1d381fe2015-07-14 16:24:56 -070049 stepResult = main.FALSE
Devin Lim58046fa2017-07-05 16:55:00 -070050 try:
51 # The variable to decide if the data should be written into data base.
52 # 1 means Yes and -1 means No.
53 main.writeData = 1
54 main.searchTerm = main.params[ 'SearchTerm' ]
55 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
56 main.dependencyPath = main.testOnDirectory + \
57 main.params[ 'DEPENDENCY' ][ 'path' ]
58 main.tsharkResultPath = main.params[ 'TsharkPath' ]
Devin Lim142b5342017-07-20 15:22:39 -070059 main.roleRequest = main.params[ 'SearchTerm' ][ 'roleRequest' ]
Devin Lim58046fa2017-07-05 16:55:00 -070060 main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
61 main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
Devin Lim58046fa2017-07-05 16:55:00 -070062 main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
63 main.topoScaleSize = len( main.topoScale )
64 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
65 wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
66 wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
67 main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] )
68 main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] )
69 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
70 main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
71 main.nodeSleep = int( main.params[ 'SLEEP' ][ 'nodeSleep' ] )
72 main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] )
73 main.MNSleep = int( main.params[ 'SLEEP' ][ 'MNsleep' ] )
74 main.pingTimeout = float( main.params[ 'TIMEOUT' ][ 'pingall' ] )
75 main.hostDiscover = main.params[ 'TOPOLOGY' ][ 'host' ]
Devin Lim142b5342017-07-20 15:22:39 -070076 main.hostDiscoverSleep = float( main.params[ 'SLEEP' ][ 'host' ] )
Devin Lima7cfdbd2017-09-29 15:02:22 -070077 main.basicMNTime = int( main.params[ 'TIMEOUT' ][ 'basicMininet' ] )
78 main.startNetTime = int( main.params[ 'TIMEOUT' ][ 'startNet' ] )
79 main.stopNetTime = int( main.params[ 'TIMEOUT' ][ 'stopNet' ] )
80 main.MNupdateTime = int( main.params[ 'TIMEOUT' ][ 'update' ] )
81 main.MNLinksTime = int( main.params[ 'TIMEOUT' ][ 'links' ] )
82 main.currScale = None
83 main.threadID = 0
Devin Lim58046fa2017-07-05 16:55:00 -070084 if main.hostDiscover == 'True':
85 main.hostDiscover = True
86 else:
87 main.hostDiscover = False
Devin Lim142b5342017-07-20 15:22:39 -070088 main.homeDir = os.path.expanduser( '~' )
Devin Lim58046fa2017-07-05 16:55:00 -070089 main.hostsData = {}
Chiyu Chengb8c2c842016-10-05 12:40:49 -070090
Devin Lim58046fa2017-07-05 16:55:00 -070091 stepResult = main.testSetUp.envSetup()
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070092 main.allinfo = {} # The dictionary to record all the data from karaf.log
Chiyu Cheng899621b2016-11-14 11:14:48 -080093
Devin Lim58046fa2017-07-05 16:55:00 -070094 for i in range( 2 ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070095 main.allinfo[ i ] = {}
96 for w in range( 3 ):
Devin Lim58046fa2017-07-05 16:55:00 -070097 # Totaltime: the time from the new switchConnection to its end
98 # swConnection: the time from the first new switchConnection to the last new switchConnection
99 # lastSwToLastRr: the time from the last new switchConnection to the last role request
100 # lastRrToLastTopology: the time form the last role request to the last topology
101 # disconnectRate: the rate that shows how many switch disconnect after connection
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700102 main.allinfo[ i ][ 'info' + str( w ) ] = { 'totalTime': 0, 'swConnection': 0, 'lastSwToLastRr': 0, 'lastRrToLastTopology': 0, 'disconnectRate': 0 }
Chiyu Cheng899621b2016-11-14 11:14:48 -0800103
Devin Lim58046fa2017-07-05 16:55:00 -0700104 main.dbFilePath = main.params[ 'DATABASE' ][ 'dbPath' ]
105 main.log.info( "Create Database file " + main.dbFilePath )
Devin Lim142b5342017-07-20 15:22:39 -0700106 resultDB = open( main.dbFilePath, 'w+' )
Devin Lim58046fa2017-07-05 16:55:00 -0700107 resultDB.close()
Chiyu Cheng899621b2016-11-14 11:14:48 -0800108
Devin Lim58046fa2017-07-05 16:55:00 -0700109 main.scaleTopoFunction = imp.load_source( wrapperFile2,
110 main.dependencyPath +
111 wrapperFile2 +
112 ".py" )
GlennRC632e2892015-10-19 18:58:41 -0700113
Devin Lim58046fa2017-07-05 16:55:00 -0700114 main.topo = imp.load_source( wrapperFile3,
115 main.dependencyPath +
116 wrapperFile3 +
117 ".py" )
GlennRC632e2892015-10-19 18:58:41 -0700118
Devin Lim58046fa2017-07-05 16:55:00 -0700119 main.ONOSbench.scp( main.Mininet1,
120 main.dependencyPath +
121 main.multiovs,
122 main.Mininet1.home,
123 direction="to" )
124 except Exception as e:
125 main.testSetUp.envSetupException( e )
126 main.testSetUp.evnSetupConclusion( stepResult )
127 main.commit = main.commit.split( " " )[ 1 ]
GlennRC632e2892015-10-19 18:58:41 -0700128
Devin Lim142b5342017-07-20 15:22:39 -0700129 def CASE2( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700130 """
131 - Set up cell
132 - Create cell file
133 - Set cell file
134 - Verify cell file
135 - Kill ONOS process
136 - Uninstall ONOS cluster
137 - Verify ONOS start up
138 - Install ONOS cluster
139 - Connect to cli
140 """
YPZhangacaaf422016-07-26 09:34:03 -0700141 import time
Devin Lima7cfdbd2017-09-29 15:02:22 -0700142 main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster )
143
144 def CASE3( self, main ):
145 """
146 cleanup mininet.
147 """
Devin Lim58046fa2017-07-05 16:55:00 -0700148 try:
149 from tests.dependencies.utils import Utils
150 except ImportError:
151 main.log.error( "Utils not found exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -0700152 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -0700153 try:
154 main.Utils
155 except ( NameError, AttributeError ):
156 main.Utils = Utils()
Devin Lima7cfdbd2017-09-29 15:02:22 -0700157 main.Utils.mininetCleanup( main.Mininet1, exitTimeout=main.basicMNTime + ( int( main.currScale ) if main.currScale is not None
158 else 1 ) * main.stopNetTime )
GlennRC632e2892015-10-19 18:58:41 -0700159
160 def CASE10( self, main ):
161 """
YPZhang85024fc2016-02-09 16:59:27 -0800162 Starting up torus topology
GlennRC632e2892015-10-19 18:58:41 -0700163 """
GlennRC475f50d2015-10-23 15:01:09 -0700164 main.case( "Starting up Mininet and verifying topology" )
165 main.caseExplanation = "Starting Mininet with a scalling topology and " +\
166 "comparing topology elements between Mininet and ONOS"
GlennRC475f50d2015-10-23 15:01:09 -0700167 if main.topoScale:
Devin Lim142b5342017-07-20 15:22:39 -0700168 main.currScale = main.topoScale.pop( 0 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700169 else:
170 main.log.error( "topology scale is empty" )
Devin Lim142b5342017-07-20 15:22:39 -0700171 main.step( "Starting up TORUS %sx%s topology" % ( main.currScale, main.currScale ) )
GlennRC475f50d2015-10-23 15:01:09 -0700172
173 main.log.info( "Constructing Mininet command" )
YPZhangacaaf422016-07-26 09:34:03 -0700174 mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs + \
175 " --switch ovsm --topo " + main.topoName + "," + main.currScale + "," + main.currScale
Devin Lim142b5342017-07-20 15:22:39 -0700176 for ctrl in main.Cluster.runningNodes:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700177 mnCmd += " --controller remote,ip=" + ctrl.ipAddress
Devin Lima7cfdbd2017-09-29 15:02:22 -0700178 stepResult = main.Mininet1.startNet( mnCmd=mnCmd, timeout=( main.basicMNTime + int( main.currScale ) * main.startNetTime ) )
GlennRC632e2892015-10-19 18:58:41 -0700179 utilities.assert_equals( expect=main.TRUE,
180 actual=stepResult,
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700181 onpass=main.topoName +
GlennRC632e2892015-10-19 18:58:41 -0700182 " topology started successfully",
183 onfail=main.topoName +
184 " topology failed to start" )
185
GlennRCe283c4b2016-01-07 13:04:10 -0800186 time.sleep( main.MNSleep )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800187 main.log.info( "Clean up Tshark" )
Devin Lim142b5342017-07-20 15:22:39 -0700188 with open( main.tsharkResultPath, "w" ) as tshark:
Chiyu Cheng899621b2016-11-14 11:14:48 -0800189 tshark.write( "" )
190 main.log.info( "Starting Tshark capture" )
191 main.ONOSbench.tsharkGrep( main.roleRequest, main.tsharkResultPath, grepOptions='-E' )
Devin Lim142b5342017-07-20 15:22:39 -0700192 main.Cluster.active( 0 ).CLI.activateApp( "org.onosproject.openflow" )
Chiyu Cheng81499422016-11-09 11:04:23 -0800193 time.sleep( main.MNSleep )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800194 main.log.info( "Stop Tshark" )
195 main.ONOSbench.tsharkStop()
196 main.log.info( "Get role request time" )
197 with open( main.tsharkResultPath, "r" ) as resultFile:
198 resultText = resultFile.readlines()
199 resultFile.close()
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700200
GlennRC475f50d2015-10-23 15:01:09 -0700201 def CASE11( self, main ):
202 """
YPZhangacaaf422016-07-26 09:34:03 -0700203 Compare topo, and sending Arping package
YPZhang85024fc2016-02-09 16:59:27 -0800204 if the topology is same, then Pass.
GlennRC475f50d2015-10-23 15:01:09 -0700205 """
206 import json
YPZhangacaaf422016-07-26 09:34:03 -0700207 import time
Devin Lim58046fa2017-07-05 16:55:00 -0700208 try:
209 from tests.dependencies.topology import Topology
210 except ImportError:
211 main.log.error( "Topology not found exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -0700212 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -0700213 try:
214 main.topoRelated
215 except ( NameError, AttributeError ):
216 main.topoRelated = Topology()
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700217 # First capture
Devin Lima7cfdbd2017-09-29 15:02:22 -0700218
219 main.postResult = True
Devin Lim4a87b2a2017-10-11 18:23:12 -0700220 main.step( "Grep information from the ONOS log" )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700221 for i in range( 3 ):
222 # Calculate total time
Devin Lim142b5342017-07-20 15:22:39 -0700223 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'end' ], 'last', index=i, funcMode='TD' )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700224 # Calculate switch connection time
Devin Lim142b5342017-07-20 15:22:39 -0700225 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'start' ], 'last', index=i, funcMode='TD' )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800226 # Calculate the time from last switch connection to the last role request
Devin Lim142b5342017-07-20 15:22:39 -0700227 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'lastSwToLastRr' ] = main.scaleTopoFunction.compareTimeDiffWithRoleRequest( main, main.searchTerm[ 'start' ], 'last', index=i )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800228 # Calculate the time from the last role request to the last topology
Devin Lim142b5342017-07-20 15:22:39 -0700229 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'lastRrToLastTopology' ] = main.scaleTopoFunction.compareTimeDiffWithRoleRequest( main, main.searchTerm[ 'end' ], 'last', index=i )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700230 # Calculate the disconnecti rate
Devin Lim142b5342017-07-20 15:22:39 -0700231 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'Disconnect' ], 'num', main.searchTerm[ 'start' ], 'num', index=i, funcMode='DR' )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700232 main.log.debug( "The data is " + str( main.allinfo[ 0 ] ) )
Devin Lima7cfdbd2017-09-29 15:02:22 -0700233 if -1 in main.allinfo[ 0 ][ 'info0' ].values() or -1 in main.allinfo[ 0 ][ 'info1' ].values() or -1 in main.allinfo[ 0 ][ 'info2' ].values():
Devin Lim4a87b2a2017-10-11 18:23:12 -0700234 utilities.assert_equals( expect=main.TRUE,
235 actual=main.FALSE,
236 onpass="Everything installed properly to the ONOS.",
237 onfail="Something happened to ONOS. Skip the rest of the steps." )
Devin Lima7cfdbd2017-09-29 15:02:22 -0700238 main.postResult = False
YPZhang81a7d4e2016-04-18 13:10:17 -0700239 else:
Devin Lima7cfdbd2017-09-29 15:02:22 -0700240 main.case( "Verifying topology: TORUS %sx%s" % ( main.currScale, main.currScale ) )
241 main.caseExplanation = "Pinging all hosts and comparing topology " +\
242 "elements between Mininet and ONOS"
243
244 main.log.info( "Gathering topology information" )
245 time.sleep( main.MNSleep )
Devin Lim4a87b2a2017-10-11 18:23:12 -0700246
Devin Lima7cfdbd2017-09-29 15:02:22 -0700247 compareRetry = 0
Devin Lim4a87b2a2017-10-11 18:23:12 -0700248 main.step( "Checking if ONOS is stable" )
249 main.scaleTopoFunction.checkingONOSStablility( main )
Devin Lima7cfdbd2017-09-29 15:02:22 -0700250
Devin Lim4a87b2a2017-10-11 18:23:12 -0700251 if main.postResult:
252 main.step( "Comparing MN topology to ONOS topology" )
Devin Lima7cfdbd2017-09-29 15:02:22 -0700253
Devin Lim4a87b2a2017-10-11 18:23:12 -0700254 compareRetry = 0
255 while compareRetry < 2:
256 stepResult = main.TRUE
257 currentDevicesResult = main.TRUE
258 currentLinksResult = main.TRUE
259 # While loop for retry
260 devices = main.topoRelated.getAll( "devices" )
261 ports = main.topoRelated.getAll( "ports" )
262 links = main.topoRelated.getAll( "links" )
263 if None in devices or None in ports or None in links:
264 main.log.warn( "Something went wrong. Retrying..." )
265 time.sleep( 20 )
Devin Lima7cfdbd2017-09-29 15:02:22 -0700266 stepResult = main.FALSE
Devin Lim4a87b2a2017-10-11 18:23:12 -0700267 compareRetry += 1
268 continue
269 mnSwitches = main.Mininet1.getSwitches( updateTimeout=main.basicMNTime + int( main.currScale ) * main.MNupdateTime )
270 main.log.info( "Comparing switches..." )
271 devicePool = []
272 for controller in range( len( main.Cluster.active() ) ):
273 t = main.Thread( target=main.topoRelated.compareDevicePort,
274 threadID=main.threadID,
275 name="Compare-Device-Port",
276 args=[ main.Mininet1, controller,
277 mnSwitches,
278 devices, ports ] )
279 devicePool.append( t )
280 t.start()
281 main.threadID = main.threadID + 1
282
283 mnLinks = main.Mininet1.getLinks( timeout=main.basicMNTime + int( main.currScale ) * main.MNLinksTime,
284 updateTimeout=main.basicMNTime + int(main.currScale) * main.MNupdateTime )
285 main.log.info( "Comparing links..." )
286 linkPool = []
287 for controller in range( len( main.Cluster.active() ) ):
288 t = main.Thread( target=main.topoRelated.compareBase,
289 threadID=main.threadID,
290 name="Compare-Link-Result",
291 args=[ links, controller,
292 main.Mininet1.compareLinks,
293 [ mnSwitches, mnLinks ] ] )
294 linkPool.append( t )
295 t.start()
296 main.threadID = main.threadID + 1
297
298 for t in devicePool:
299 t.join()
300 currentDevicesResult = currentDevicesResult and t.result
301 for t in linkPool:
302 t.join()
303 currentLinksResult = currentLinksResult and t.result
304 stepResult = stepResult and currentDevicesResult and currentLinksResult
305 if stepResult:
306 break
307 compareRetry += 1
308 utilities.assert_equals( expect=main.TRUE,
309 actual=stepResult,
310 onpass=" Topology match Mininet",
311 onfail="ONOS Topology doesn't match Mininet" )
312 main.scaleTopoFunction.checkingONOSStablility( main )
313 if stepResult and main.postResult:
314 if main.hostDiscover:
315 hostList = []
316 for i in range( 1, int( main.currScale ) + 1 ):
317 for j in range( 1, int( main.currScale ) + 1 ):
318 # Generate host list
319 hoststr = "h" + str( i ) + "x" + str( j )
320 hostList.append( hoststr )
321 for i in range( len( hostList ) ):
322 main.topo.sendArpPackage( main, hostList[ i ] )
323 time.sleep( 20 )
324 totalHost = main.topo.getHostNum( main )
325 if totalHost == int( main.currScale ) * int( main.currScale ):
326 main.log.info( "Discovered all hosts" )
327 stepResult = stepResult and main.TRUE
328 else:
329 main.log.warn( "Some hosts ware not discovered by ONOS... Topology doesn't match!" )
330 stepResult = main.FALSE
331 utilities.assert_equals( expect=main.TRUE,
332 actual=stepResult,
333 onpass=" Topology match Mininet",
334 onfail="ONOS Topology doesn't match Mininet" )
335 main.log.info( "Finished this iteration, continue to scale next topology." )
336 else:
Devin Lima7cfdbd2017-09-29 15:02:22 -0700337 utilities.assert_equals( expect=main.TRUE,
Devin Lim4a87b2a2017-10-11 18:23:12 -0700338 actual=main.FALSE,
339 onpass="ONOS is stable.",
340 onfail="Something happened to ONOS. Skip the rest of the steps." )
341 main.postResult = False
GlennRC475f50d2015-10-23 15:01:09 -0700342
GlennRC632e2892015-10-19 18:58:41 -0700343 def CASE100( self, main ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700344 """
YPZhang81a7d4e2016-04-18 13:10:17 -0700345 Bring Down node 3
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700346 """
347 main.case( "Bring ONOS node 3 down: TORUS %sx%s" % ( main.currScale, main.currScale ) )
GlennRC475f50d2015-10-23 15:01:09 -0700348 main.caseExplanation = "Balance masters to make sure " +\
349 "each controller has some devices and " +\
350 "stop ONOS node 3 service. "
351
GlennRC475f50d2015-10-23 15:01:09 -0700352 stepResult = main.FALSE
GlennRCed2122e2015-10-21 14:38:46 -0700353 main.step( "Bringing down node 3" )
GlennRCed2122e2015-10-21 14:38:46 -0700354 # Always bring down the third node
355 main.deadNode = 2
GlennRCed2122e2015-10-21 14:38:46 -0700356 # Printing purposes
GlennRC632e2892015-10-19 18:58:41 -0700357 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700358 main.log.info( "Stopping node %s" % node )
Devin Lim142b5342017-07-20 15:22:39 -0700359 stepResult = main.ONOSbench.onosStop( main.Cluster.active( main.deadNode ).ipAddress )
GlennRC475f50d2015-10-23 15:01:09 -0700360 main.log.info( "Removing dead node from list of active nodes" )
Devin Lim142b5342017-07-20 15:22:39 -0700361 main.Cluster.runningNodes[ main.deadNode ].active = False
GlennRC632e2892015-10-19 18:58:41 -0700362
YPZhang77badfc2016-03-09 10:28:59 -0800363 utilities.assert_equals( expect=main.TRUE,
364 actual=stepResult,
365 onpass="Successfully bring down node 3",
366 onfail="Failed to bring down node 3" )
GlennRC632e2892015-10-19 18:58:41 -0700367
GlennRC475f50d2015-10-23 15:01:09 -0700368 def CASE200( self, main ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700369 """
YPZhangacaaf422016-07-26 09:34:03 -0700370 Bring up onos node
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700371 """
372 main.case( "Bring ONOS node 3 up: TORUS %sx%s" % ( main.currScale, main.currScale ) )
GlennRC475f50d2015-10-23 15:01:09 -0700373 main.caseExplanation = "Bring node 3 back up and balance the masters"
Devin Lim142b5342017-07-20 15:22:39 -0700374 ctrl = main.Cluster.runningNodes[ main.deadNode ]
GlennRC632e2892015-10-19 18:58:41 -0700375 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700376 main.log.info( "Starting node %s" % node )
Devin Lim142b5342017-07-20 15:22:39 -0700377 stepResult = main.ONOSbench.onosStart( ctrl.ipAddress )
GlennRC632e2892015-10-19 18:58:41 -0700378 main.log.info( "Starting onos cli" )
Devin Lim142b5342017-07-20 15:22:39 -0700379 stepResult = stepResult and \
380 ctrl.CLI.startOnosCli( ctrl.ipAddress )
GlennRC475f50d2015-10-23 15:01:09 -0700381 main.log.info( "Adding previously dead node to list of active nodes" )
Devin Lim142b5342017-07-20 15:22:39 -0700382 ctrl.active = True
GlennRC632e2892015-10-19 18:58:41 -0700383
GlennRC632e2892015-10-19 18:58:41 -0700384 utilities.assert_equals( expect=main.TRUE,
385 actual=stepResult,
386 onpass="Successfully brought up onos node %s" % node,
387 onfail="Failed to bring up onos node %s" % node )
388
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700389 time.sleep( main.nodeSleep )
GlennRCe283c4b2016-01-07 13:04:10 -0800390
391 def CASE300( self, main ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700392 """
GlennRCe283c4b2016-01-07 13:04:10 -0800393 Balancing Masters
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700394 """
395 time.sleep( main.balanceSleep )
GlennRC475f50d2015-10-23 15:01:09 -0700396 main.step( "Balancing Masters" )
GlennRCe283c4b2016-01-07 13:04:10 -0800397
GlennRC475f50d2015-10-23 15:01:09 -0700398 stepResult = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700399 if main.Cluster.active():
400 stepResult = utilities.retry( main.Cluster.next().CLI.balanceMasters,
YPZhang924ccfe2016-01-26 14:17:30 -0800401 main.FALSE,
402 [],
403 sleep=3,
404 attempts=3 )
GlennRCe283c4b2016-01-07 13:04:10 -0800405 else:
406 main.log.error( "List of active nodes is empty" )
GlennRC475f50d2015-10-23 15:01:09 -0700407 utilities.assert_equals( expect=main.TRUE,
408 actual=stepResult,
409 onpass="Balance masters was successfull",
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700410 onfail="Failed to balance masters" )
Devin Lim142b5342017-07-20 15:22:39 -0700411 time.sleep( main.balanceSleep )
GlennRC475f50d2015-10-23 15:01:09 -0700412
GlennRC632e2892015-10-19 18:58:41 -0700413 def CASE1000( self, main ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700414 """
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700415 Report errors/warnings/exceptions
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700416 """
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700417 # Compare the slowest Node through total time of each node
Devin Lima7cfdbd2017-09-29 15:02:22 -0700418 if main.postResult:
419 slowestNode = 0
420 slowestTotalTime = 0
421 # Second capture
422 for i in range( 3 ):
423 # Calculate total time
424 main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main,
425 main.searchTerm[ 'start' ],
426 'first',
427 main.searchTerm[ 'end' ],
428 'last',
429 index=i,
430 funcMode='TD' )
431 # Compare the total time
432 if main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ] > slowestTotalTime:
433 slowestTotalTime = main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ]
434 slowestNode = i
435 # Calculate switch connection time
436 main.allinfo[ 1 ][ 'info' + str( i ) ][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main,
437 main.searchTerm[ 'start' ],
438 'first',
439 main.searchTerm[ 'start' ],
440 'last',
441 index=i,
442 funcMode='TD' )
443 # Calculate the time from last switch connection to the last role request
444 main.allinfo[ 1 ][ 'info' + str( i ) ][ 'lastSwToLastRr' ] = main.scaleTopoFunction.compareTimeDiffWithRoleRequest( main,
445 main.searchTerm[ 'start' ],
446 'last',
447 index=i )
448 # Calculate the time from the last role request to the last topology
449 main.allinfo[ 1 ][ 'info' + str( i ) ][ 'lastRrToLastTopology' ] = main.scaleTopoFunction.compareTimeDiffWithRoleRequest( main,
450 main.searchTerm[ 'end' ],
451 'last',
452 index=i )
453 # Calculate the disconnecti rate
454 main.allinfo[ 1 ][ 'info' + str( i ) ][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main,
455 main.searchTerm[ 'Disconnect' ],
456 'num',
457 main.searchTerm[ 'start' ],
458 'num',
459 index=i,
460 funcMode='DR' )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700461
Devin Lima7cfdbd2017-09-29 15:02:22 -0700462 if ( main.allinfo[ 0 ] != main.allinfo[ 1 ] ):
463 main.log.error( "The results of two capture are different!" )
464 main.log.debug( "The data is " + str( main.allinfo ) )
465 if main.writeData != -1:
466 main.log.info( "Write the date into database" )
467 # write the date into data base
468 with open( main.dbFilePath, "a" ) as dbFile:
469 temp = str( main.currScale )
470 temp += ",'baremetal1'"
471 # put result from second capture into data base
472 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode ) ][ 'totalTime' ] )
473 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode ) ][ 'swConnection' ] )
474 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode ) ][ 'lastSwToLastRr' ] )
475 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode ) ][ 'lastRrToLastTopology' ] )
476 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode ) ][ 'disconnectRate' ] )
477 temp += "\n"
478 dbFile.write( temp )
479 else:
480 main.log.error( "The data from log is wrong!" )
481 main.writeData = 1
482 main.case( "Checking logs for errors, warnings, and exceptions" )
483 main.log.info( "Error report: \n" )
484 main.ONOSbench.logReport( main.Cluster.active( 0 ).ipAddress,
485 [ "INFO",
486 "FOLLOWER",
487 "WARN",
488 "flow",
489 "ERROR",
490 "Except" ],
491 "s" )