blob: 71758f6ff3b5b7d60b4453e358c6fdacbfd694b8 [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
2Copyright 2015 Open Networking Foundation (ONF)
3
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
11 (at your option) any later version.
12
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
kelvin-onlab1d381fe2015-07-14 16:24:56 -070021
22# Testing network scalability, this test suite scales up a network topology
23# using mininet and verifies ONOS stability
24
GlennRC1c5df3c2015-08-27 16:12:09 -070025class SCPFscaleTopo:
kelvin-onlab1d381fe2015-07-14 16:24:56 -070026
27 def __init__( self ):
28 self.default = ''
29
30 def CASE1( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -070031 import os
32 import imp
kelvin-onlab1d381fe2015-07-14 16:24:56 -070033 """
34 - Construct tests variables
35 - GIT ( optional )
36 - Checkout ONOS master branch
37 - Pull latest ONOS code
38 - Building ONOS ( optional )
39 - Install ONOS package
40 - Build ONOS package
41 """
Devin Lim58046fa2017-07-05 16:55:00 -070042 try:
43 from tests.dependencies.ONOSSetup import ONOSSetup
44 main.testSetUp = ONOSSetup()
45 except ImportError:
46 main.log.error( "ONOSSetup not found. exiting the test" )
47 main.exit()
48 main.testSetUp.envSetupDescription()
kelvin-onlab1d381fe2015-07-14 16:24:56 -070049 stepResult = main.FALSE
Devin Lim58046fa2017-07-05 16:55:00 -070050 try:
51 # The variable to decide if the data should be written into data base.
52 # 1 means Yes and -1 means No.
53 main.writeData = 1
54 main.searchTerm = main.params[ 'SearchTerm' ]
55 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
56 main.dependencyPath = main.testOnDirectory + \
57 main.params[ 'DEPENDENCY' ][ 'path' ]
58 main.tsharkResultPath = main.params[ 'TsharkPath' ]
Devin Lim142b5342017-07-20 15:22:39 -070059 main.roleRequest = main.params[ 'SearchTerm' ][ 'roleRequest' ]
Devin Lim58046fa2017-07-05 16:55:00 -070060 main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
61 main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
Devin Lim58046fa2017-07-05 16:55:00 -070062 main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
63 main.topoScaleSize = len( main.topoScale )
64 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
65 wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
66 wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
67 main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] )
68 main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] )
69 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
70 main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
71 main.nodeSleep = int( main.params[ 'SLEEP' ][ 'nodeSleep' ] )
72 main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] )
73 main.MNSleep = int( main.params[ 'SLEEP' ][ 'MNsleep' ] )
74 main.pingTimeout = float( main.params[ 'TIMEOUT' ][ 'pingall' ] )
75 main.hostDiscover = main.params[ 'TOPOLOGY' ][ 'host' ]
Devin Lim142b5342017-07-20 15:22:39 -070076 main.hostDiscoverSleep = float( main.params[ 'SLEEP' ][ 'host' ] )
Devin Lim58046fa2017-07-05 16:55:00 -070077 if main.hostDiscover == 'True':
78 main.hostDiscover = True
79 else:
80 main.hostDiscover = False
Devin Lim142b5342017-07-20 15:22:39 -070081 main.homeDir = os.path.expanduser( '~' )
Devin Lim58046fa2017-07-05 16:55:00 -070082 main.hostsData = {}
Chiyu Chengb8c2c842016-10-05 12:40:49 -070083
Chiyu Cheng899621b2016-11-14 11:14:48 -080084
Devin Lim58046fa2017-07-05 16:55:00 -070085 stepResult = main.testSetUp.envSetup()
86 main.allinfo = {} # The dictionary to record all the data from karaf.log
Chiyu Cheng899621b2016-11-14 11:14:48 -080087
Devin Lim58046fa2017-07-05 16:55:00 -070088 for i in range( 2 ):
89 main.allinfo[ i ]={}
90 for w in range ( 3 ):
91 # Totaltime: the time from the new switchConnection to its end
92 # swConnection: the time from the first new switchConnection to the last new switchConnection
93 # lastSwToLastRr: the time from the last new switchConnection to the last role request
94 # lastRrToLastTopology: the time form the last role request to the last topology
95 # disconnectRate: the rate that shows how many switch disconnect after connection
96 main.allinfo[ i ][ 'info' + str( w ) ]= { 'totalTime': 0, 'swConnection': 0, 'lastSwToLastRr': 0, 'lastRrToLastTopology': 0, 'disconnectRate': 0 }
Chiyu Cheng899621b2016-11-14 11:14:48 -080097
Devin Lim58046fa2017-07-05 16:55:00 -070098 main.dbFilePath = main.params[ 'DATABASE' ][ 'dbPath' ]
99 main.log.info( "Create Database file " + main.dbFilePath )
Devin Lim142b5342017-07-20 15:22:39 -0700100 resultDB = open( main.dbFilePath, 'w+' )
Devin Lim58046fa2017-07-05 16:55:00 -0700101 resultDB.close()
Chiyu Cheng899621b2016-11-14 11:14:48 -0800102
Devin Lim58046fa2017-07-05 16:55:00 -0700103 main.scaleTopoFunction = imp.load_source( wrapperFile2,
104 main.dependencyPath +
105 wrapperFile2 +
106 ".py" )
GlennRC632e2892015-10-19 18:58:41 -0700107
Devin Lim58046fa2017-07-05 16:55:00 -0700108 main.topo = imp.load_source( wrapperFile3,
109 main.dependencyPath +
110 wrapperFile3 +
111 ".py" )
GlennRC632e2892015-10-19 18:58:41 -0700112
Devin Lim58046fa2017-07-05 16:55:00 -0700113 main.ONOSbench.scp( main.Mininet1,
114 main.dependencyPath +
115 main.multiovs,
116 main.Mininet1.home,
117 direction="to" )
118 except Exception as e:
119 main.testSetUp.envSetupException( e )
120 main.testSetUp.evnSetupConclusion( stepResult )
121 main.commit = main.commit.split( " " )[ 1 ]
GlennRC632e2892015-10-19 18:58:41 -0700122
Devin Lim142b5342017-07-20 15:22:39 -0700123 def CASE2( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700124 """
125 - Set up cell
126 - Create cell file
127 - Set cell file
128 - Verify cell file
129 - Kill ONOS process
130 - Uninstall ONOS cluster
131 - Verify ONOS start up
132 - Install ONOS cluster
133 - Connect to cli
134 """
YPZhangacaaf422016-07-26 09:34:03 -0700135 import time
Devin Lim58046fa2017-07-05 16:55:00 -0700136 try:
137 from tests.dependencies.utils import Utils
138 except ImportError:
139 main.log.error( "Utils not found exiting the test" )
140 main.exit()
141 try:
142 main.Utils
143 except ( NameError, AttributeError ):
144 main.Utils = Utils()
145 main.Utils.mininetCleanup( main.Mininet1 )
Devin Lim142b5342017-07-20 15:22:39 -0700146 main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster )
GlennRC632e2892015-10-19 18:58:41 -0700147
148 def CASE10( self, main ):
149 """
YPZhang85024fc2016-02-09 16:59:27 -0800150 Starting up torus topology
GlennRC632e2892015-10-19 18:58:41 -0700151 """
GlennRC475f50d2015-10-23 15:01:09 -0700152
153 main.case( "Starting up Mininet and verifying topology" )
154 main.caseExplanation = "Starting Mininet with a scalling topology and " +\
155 "comparing topology elements between Mininet and ONOS"
GlennRC475f50d2015-10-23 15:01:09 -0700156 if main.topoScale:
Devin Lim142b5342017-07-20 15:22:39 -0700157 main.currScale = main.topoScale.pop( 0 )
GlennRC475f50d2015-10-23 15:01:09 -0700158 else: main.log.error( "topology scale is empty" )
Devin Lim142b5342017-07-20 15:22:39 -0700159 main.step( "Starting up TORUS %sx%s topology" % ( main.currScale, main.currScale ) )
GlennRC475f50d2015-10-23 15:01:09 -0700160
161 main.log.info( "Constructing Mininet command" )
YPZhangacaaf422016-07-26 09:34:03 -0700162 mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs + \
163 " --switch ovsm --topo " + main.topoName + "," + main.currScale + "," + main.currScale
Devin Lim142b5342017-07-20 15:22:39 -0700164 for ctrl in main.Cluster.runningNodes:
165 mnCmd += " --controller remote,ip=" + ctrl.ipAddress
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700166 stepResult = main.Mininet1.startNet( mnCmd=mnCmd )
GlennRC632e2892015-10-19 18:58:41 -0700167 utilities.assert_equals( expect=main.TRUE,
168 actual=stepResult,
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700169 onpass=main.topoName +
GlennRC632e2892015-10-19 18:58:41 -0700170 " topology started successfully",
171 onfail=main.topoName +
172 " topology failed to start" )
173
GlennRCe283c4b2016-01-07 13:04:10 -0800174 time.sleep( main.MNSleep )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800175 main.log.info( "Clean up Tshark" )
Devin Lim142b5342017-07-20 15:22:39 -0700176 with open( main.tsharkResultPath, "w" ) as tshark:
Chiyu Cheng899621b2016-11-14 11:14:48 -0800177 tshark.write( "" )
178 main.log.info( "Starting Tshark capture" )
179 main.ONOSbench.tsharkGrep( main.roleRequest, main.tsharkResultPath, grepOptions='-E' )
Devin Lim142b5342017-07-20 15:22:39 -0700180 main.Cluster.active( 0 ).CLI.activateApp( "org.onosproject.openflow" )
Chiyu Cheng81499422016-11-09 11:04:23 -0800181 time.sleep( main.MNSleep )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800182 main.log.info( "Stop Tshark" )
183 main.ONOSbench.tsharkStop()
184 main.log.info( "Get role request time" )
185 with open( main.tsharkResultPath, "r" ) as resultFile:
186 resultText = resultFile.readlines()
187 resultFile.close()
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700188
GlennRC475f50d2015-10-23 15:01:09 -0700189 def CASE11( self, main ):
190 """
YPZhangacaaf422016-07-26 09:34:03 -0700191 Compare topo, and sending Arping package
YPZhang85024fc2016-02-09 16:59:27 -0800192 if the topology is same, then Pass.
GlennRC475f50d2015-10-23 15:01:09 -0700193 """
194 import json
YPZhangacaaf422016-07-26 09:34:03 -0700195 import time
Devin Lim58046fa2017-07-05 16:55:00 -0700196 try:
197 from tests.dependencies.topology import Topology
198 except ImportError:
199 main.log.error( "Topology not found exiting the test" )
200 main.exit()
201 try:
202 main.topoRelated
203 except ( NameError, AttributeError ):
204 main.topoRelated = Topology()
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700205 # First capture
206 for i in range( 3 ):
207 # Calculate total time
Devin Lim142b5342017-07-20 15:22:39 -0700208 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'end' ], 'last', index=i, funcMode='TD' )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700209 # Calculate switch connection time
Devin Lim142b5342017-07-20 15:22:39 -0700210 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'start' ], 'last', index=i, funcMode='TD' )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800211 # Calculate the time from last switch connection to the last role request
Devin Lim142b5342017-07-20 15:22:39 -0700212 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'lastSwToLastRr' ] = main.scaleTopoFunction.compareTimeDiffWithRoleRequest( main, main.searchTerm[ 'start' ], 'last', index=i )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800213 # Calculate the time from the last role request to the last topology
Devin Lim142b5342017-07-20 15:22:39 -0700214 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'lastRrToLastTopology' ] = main.scaleTopoFunction.compareTimeDiffWithRoleRequest( main, main.searchTerm[ 'end' ], 'last', index=i )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700215 # Calculate the disconnecti rate
Devin Lim142b5342017-07-20 15:22:39 -0700216 main.allinfo[ 0 ][ 'info' + str( i ) ][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'Disconnect' ], 'num', main.searchTerm[ 'start' ], 'num', index=i, funcMode='DR' )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700217 main.log.debug( "The data is " + str( main.allinfo[ 0 ] ) )
GlennRC475f50d2015-10-23 15:01:09 -0700218
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700219 main.case( "Verifying topology: TORUS %sx%s" % ( main.currScale, main.currScale ) )
YPZhang85024fc2016-02-09 16:59:27 -0800220 main.caseExplanation = "Pinging all hosts and comparing topology " +\
GlennRC475f50d2015-10-23 15:01:09 -0700221 "elements between Mininet and ONOS"
GlennRCe283c4b2016-01-07 13:04:10 -0800222
Devin Lim142b5342017-07-20 15:22:39 -0700223 main.log.info( "Gathering topology information" )
YPZhang85024fc2016-02-09 16:59:27 -0800224 time.sleep( main.MNSleep )
YPZhang85024fc2016-02-09 16:59:27 -0800225 stepResult = main.TRUE
GlennRC475f50d2015-10-23 15:01:09 -0700226 main.step( "Comparing MN topology to ONOS topology" )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700227 compareRetry = 0
228 while compareRetry < 3:
YPZhang81a7d4e2016-04-18 13:10:17 -0700229 #While loop for retry
Devin Lim142b5342017-07-20 15:22:39 -0700230 devices = main.topoRelated.getAll( "devices" )
231 ports = main.topoRelated.getAll( "ports" )
232 links = main.topoRelated.getAll( "links" )
YPZhang81a7d4e2016-04-18 13:10:17 -0700233 mnSwitches = main.Mininet1.getSwitches()
Devin Lim142b5342017-07-20 15:22:39 -0700234 mnLinks = main.Mininet1.getLinks( timeout=180 )
GlennRC475f50d2015-10-23 15:01:09 -0700235
Devin Lim142b5342017-07-20 15:22:39 -0700236 for controller in range( len( main.Cluster.active() ) ):
Devin Lim58046fa2017-07-05 16:55:00 -0700237 currentDevicesResult = main.topoRelated.compareDevicePort(
238 main.Mininet1, controller,
239 mnSwitches,
240 devices, ports )
YPZhang81a7d4e2016-04-18 13:10:17 -0700241
Devin Lim58046fa2017-07-05 16:55:00 -0700242 currentLinksResult = main.topoRelated.compareBase( links, controller,
243 main.Mininet1.compareLinks,
244 [ mnSwitches, mnLinks ] )
YPZhang81a7d4e2016-04-18 13:10:17 -0700245
YPZhangacaaf422016-07-26 09:34:03 -0700246 stepResult = stepResult and currentDevicesResult and currentLinksResult
YPZhang81a7d4e2016-04-18 13:10:17 -0700247 if stepResult:
248 break
249 compareRetry += 1
Devin Lim142b5342017-07-20 15:22:39 -0700250 utilities.assert_equals( expect=main.TRUE,
251 actual=stepResult,
252 onpass=" Topology match Mininet",
253 onfail="ONOS Topology doesn't match Mininet")
YPZhang81a7d4e2016-04-18 13:10:17 -0700254
YPZhangacaaf422016-07-26 09:34:03 -0700255 if stepResult:
256 if main.hostDiscover:
257 hostList = []
258 for i in range( 1, int( main.currScale ) + 1 ):
259 for j in range( 1, int( main.currScale ) + 1) :
260 # Generate host list
261 hoststr = "h" + str(i) + "x" + str(j)
262 hostList.append(hoststr)
263 for i in range( len(hostList) ):
264 totalHost = main.topo.sendArpPackage( main, hostList[i] )
265 time.sleep( main.hostDiscoverSleep )
266 if totalHost < 0:
267 # if totalHost less than 0 which means dependence function has exception.
268 main.log.info( "Error when discover host!" )
269 break
270 if totalHost == int( main.currScale ) * int( main.currScale ):
271 main.log.info( "Discovered all hosts" )
You Wang7bb9c462016-08-10 14:18:16 -0700272 stepResult = stepResult and main.TRUE
YPZhangacaaf422016-07-26 09:34:03 -0700273 else:
274 main.log.warn( "Some hosts ware not discovered by ONOS... Topology doesn't match!" )
You Wang7bb9c462016-08-10 14:18:16 -0700275 stepResult = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700276 utilities.assert_equals( expect=main.TRUE,
277 actual=stepResult,
278 onpass=" Topology match Mininet",
279 onfail="ONOS Topology doesn't match Mininet")
YPZhangacaaf422016-07-26 09:34:03 -0700280 main.log.info( "Finished this iteration, continue to scale next topology." )
YPZhang81a7d4e2016-04-18 13:10:17 -0700281 else:
YPZhangacaaf422016-07-26 09:34:03 -0700282 main.log.info( "Clean up and exit TestON. Finished this test." )
283 main.cleanup()
284 main.exit()
GlennRC475f50d2015-10-23 15:01:09 -0700285
GlennRC632e2892015-10-19 18:58:41 -0700286 def CASE100( self, main ):
287 '''
YPZhang81a7d4e2016-04-18 13:10:17 -0700288 Bring Down node 3
GlennRC632e2892015-10-19 18:58:41 -0700289 '''
GlennRC475f50d2015-10-23 15:01:09 -0700290
YPZhangacaaf422016-07-26 09:34:03 -0700291 main.case("Bring ONOS node 3 down: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700292 main.caseExplanation = "Balance masters to make sure " +\
293 "each controller has some devices and " +\
294 "stop ONOS node 3 service. "
295
GlennRC475f50d2015-10-23 15:01:09 -0700296 stepResult = main.FALSE
GlennRCed2122e2015-10-21 14:38:46 -0700297 main.step( "Bringing down node 3" )
GlennRCed2122e2015-10-21 14:38:46 -0700298 # Always bring down the third node
299 main.deadNode = 2
GlennRCed2122e2015-10-21 14:38:46 -0700300 # Printing purposes
GlennRC632e2892015-10-19 18:58:41 -0700301 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700302 main.log.info( "Stopping node %s" % node )
Devin Lim142b5342017-07-20 15:22:39 -0700303 stepResult = main.ONOSbench.onosStop( main.Cluster.active( main.deadNode ).ipAddress )
GlennRC475f50d2015-10-23 15:01:09 -0700304 main.log.info( "Removing dead node from list of active nodes" )
Devin Lim142b5342017-07-20 15:22:39 -0700305 main.Cluster.runningNodes[ main.deadNode ].active = False
GlennRC632e2892015-10-19 18:58:41 -0700306
YPZhang77badfc2016-03-09 10:28:59 -0800307 utilities.assert_equals( expect=main.TRUE,
308 actual=stepResult,
309 onpass="Successfully bring down node 3",
310 onfail="Failed to bring down node 3" )
GlennRC632e2892015-10-19 18:58:41 -0700311
GlennRC475f50d2015-10-23 15:01:09 -0700312 def CASE200( self, main ):
GlennRC632e2892015-10-19 18:58:41 -0700313 '''
YPZhangacaaf422016-07-26 09:34:03 -0700314 Bring up onos node
GlennRC632e2892015-10-19 18:58:41 -0700315 '''
GlennRC475f50d2015-10-23 15:01:09 -0700316
YPZhangacaaf422016-07-26 09:34:03 -0700317 main.case("Bring ONOS node 3 up: TORUS %sx%s" % (main.currScale, main.currScale))
GlennRC475f50d2015-10-23 15:01:09 -0700318 main.caseExplanation = "Bring node 3 back up and balance the masters"
Devin Lim142b5342017-07-20 15:22:39 -0700319 ctrl = main.Cluster.runningNodes[ main.deadNode ]
GlennRC632e2892015-10-19 18:58:41 -0700320 node = main.deadNode + 1
GlennRC632e2892015-10-19 18:58:41 -0700321 main.log.info( "Starting node %s" % node )
Devin Lim142b5342017-07-20 15:22:39 -0700322 stepResult = main.ONOSbench.onosStart( ctrl.ipAddress )
GlennRC632e2892015-10-19 18:58:41 -0700323 main.log.info( "Starting onos cli" )
Devin Lim142b5342017-07-20 15:22:39 -0700324 stepResult = stepResult and \
325 ctrl.CLI.startOnosCli( ctrl.ipAddress )
GlennRC475f50d2015-10-23 15:01:09 -0700326 main.log.info( "Adding previously dead node to list of active nodes" )
Devin Lim142b5342017-07-20 15:22:39 -0700327 ctrl.active = True
GlennRC632e2892015-10-19 18:58:41 -0700328
GlennRC632e2892015-10-19 18:58:41 -0700329 utilities.assert_equals( expect=main.TRUE,
330 actual=stepResult,
331 onpass="Successfully brought up onos node %s" % node,
332 onfail="Failed to bring up onos node %s" % node )
333
334
GlennRCe283c4b2016-01-07 13:04:10 -0800335 time.sleep(main.nodeSleep)
336
337 def CASE300( self, main ):
338 '''
339
340 Balancing Masters
341 '''
YPZhang85024fc2016-02-09 16:59:27 -0800342 time.sleep(main.balanceSleep)
GlennRC475f50d2015-10-23 15:01:09 -0700343 main.step( "Balancing Masters" )
GlennRCe283c4b2016-01-07 13:04:10 -0800344
GlennRC475f50d2015-10-23 15:01:09 -0700345 stepResult = main.FALSE
Devin Lim142b5342017-07-20 15:22:39 -0700346 if main.Cluster.active():
347 stepResult = utilities.retry( main.Cluster.next().CLI.balanceMasters,
YPZhang924ccfe2016-01-26 14:17:30 -0800348 main.FALSE,
349 [],
350 sleep=3,
351 attempts=3 )
GlennRCe283c4b2016-01-07 13:04:10 -0800352 else:
353 main.log.error( "List of active nodes is empty" )
GlennRC475f50d2015-10-23 15:01:09 -0700354 utilities.assert_equals( expect=main.TRUE,
355 actual=stepResult,
356 onpass="Balance masters was successfull",
357 onfail="Failed to balance masters")
Devin Lim142b5342017-07-20 15:22:39 -0700358 time.sleep( main.balanceSleep )
GlennRC475f50d2015-10-23 15:01:09 -0700359
GlennRC632e2892015-10-19 18:58:41 -0700360 def CASE1000( self, main ):
kelvin-onlab1d381fe2015-07-14 16:24:56 -0700361 '''
362 Report errors/warnings/exceptions
363 '''
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700364 # Compare the slowest Node through total time of each node
365 slowestNode = 0
366 slowestTotalTime = 0
367 # Second capture
368 for i in range( 3 ):
369 # Calculate total time
Devin Lim142b5342017-07-20 15:22:39 -0700370 main.allinfo[ 1 ][ 'info' + str( i )][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main,
371 main.searchTerm[ 'start' ],
372 'first',
373 main.searchTerm[ 'end' ],
374 'last',
375 index=i,
376 funcMode='TD' )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700377 # Compare the total time
378 if main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ] > slowestTotalTime:
379 slowestTotalTime = main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ]
380 slowestNode = i
381 # Calculate switch connection time
Devin Lim142b5342017-07-20 15:22:39 -0700382 main.allinfo[ 1 ][ 'info' + str( i )][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main,
383 main.searchTerm[ 'start' ],
384 'first',
385 main.searchTerm[ 'start' ],
386 'last',
387 index=i,
388 funcMode='TD' )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800389 # Calculate the time from last switch connection to the last role request
Devin Lim142b5342017-07-20 15:22:39 -0700390 main.allinfo[ 1 ][ 'info' + str( i )][ 'lastSwToLastRr' ] = main.scaleTopoFunction.compareTimeDiffWithRoleRequest( main,
391 main.searchTerm[ 'start' ],
392 'last',
393 index=i )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800394 # Calculate the time from the last role request to the last topology
Devin Lim142b5342017-07-20 15:22:39 -0700395 main.allinfo[ 1 ][ 'info' + str( i )][ 'lastRrToLastTopology' ] = main.scaleTopoFunction.compareTimeDiffWithRoleRequest( main,
396 main.searchTerm[ 'end' ],
397 'last',
398 index=i )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700399 # Calculate the disconnecti rate
Devin Lim142b5342017-07-20 15:22:39 -0700400 main.allinfo[ 1 ][ 'info' + str( i )][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main,
401 main.searchTerm[ 'Disconnect' ],
402 'num',
403 main.searchTerm[ 'start' ],
404 'num',
405 index=i,
406 funcMode='DR' )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700407
408 if ( main.allinfo[ 0 ] != main.allinfo[ 1 ] ):
409 main.log.error( "The results of two capture are different!" )
410 main.log.debug( "The data is " + str( main.allinfo ) )
411 if main.writeData != -1:
412 main.log.info( "Write the date into database" )
413 # write the date into data base
414 with open( main.dbFilePath, "a" ) as dbFile:
415 temp = str( main.currScale )
416 temp += ",'baremetal1'"
417 # put result from second capture into data base
418 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'totalTime' ] )
419 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'swConnection' ] )
Chiyu Cheng899621b2016-11-14 11:14:48 -0800420 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'lastSwToLastRr' ] )
421 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'lastRrToLastTopology' ] )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700422 temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'disconnectRate' ] )
423 temp += "\n"
424 dbFile.write( temp )
425 else:
426 main.log.error( "The data from log is wrong!" )
427 main.writeData = 1
GlennRC475f50d2015-10-23 15:01:09 -0700428 main.case( "Checking logs for errors, warnings, and exceptions" )
Chiyu Chengb8c2c842016-10-05 12:40:49 -0700429 main.log.info( "Error report: \n" )
Devin Lim142b5342017-07-20 15:22:39 -0700430 main.ONOSbench.logReport( main.Cluster.active( 0 ).ipAddress,
GlennRC475f50d2015-10-23 15:01:09 -0700431 [ "INFO",
432 "FOLLOWER",
433 "WARN",
434 "flow",
435 "ERROR",
436 "Except" ],
Chiyu Cheng899621b2016-11-14 11:14:48 -0800437 "s" )