blob: f542554a6e392a9fe2105032ecd720fe688cc82f [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2016 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jon Hall1efcb3f2016-08-23 13:42:15 -070021import os
22import imp
23import time
24import json
25import urllib
26from core import utilities
27
28
29class Testcaselib:
Pierfb719b12016-09-19 14:51:44 -070030
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070031 useSSH = True
Pierfb719b12016-09-19 14:51:44 -070032
Jon Hall1efcb3f2016-08-23 13:42:15 -070033 @staticmethod
34 def initTest( main ):
35 """
36 - Construct tests variables
37 - GIT ( optional )
38 - Checkout ONOS master branch
39 - Pull latest ONOS code
40 - Building ONOS ( optional )
41 - Install ONOS package
42 - Build ONOS package
43 """
Devin Lim58046fa2017-07-05 16:55:00 -070044 try:
45 from tests.dependencies.ONOSSetup import ONOSSetup
46 main.testSetUp = ONOSSetup()
47 except ImportError:
48 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070049 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -070050 main.testSetUp.envSetupDescription()
51 stepResult = main.FALSE
52 try:
53 main.step( "Constructing test variables" )
54 # Test variables
55 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
56 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
57 main.diff = main.params[ 'ENV' ][ 'diffApps' ]
58 main.path = os.path.dirname( main.testFile )
59 main.dependencyPath = main.path + "/../dependencies/"
60 main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
61 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070062 main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
Devin Lim58046fa2017-07-05 16:55:00 -070063 main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
64 # main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
65 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
66 # -- INIT SECTION, ONLY RUNS ONCE -- #
Jon Hall1efcb3f2016-08-23 13:42:15 -070067
Devin Lim58046fa2017-07-05 16:55:00 -070068 copyResult1 = main.ONOSbench.scp( main.Mininet1,
69 main.dependencyPath +
70 main.topology,
71 main.Mininet1.home,
72 direction="to" )
Devin Lim142b5342017-07-20 15:22:39 -070073 stepResult = main.testSetUp.envSetup()
Devin Lim58046fa2017-07-05 16:55:00 -070074 except Exception as e:
75 main.testSetUp.envSetupException( e )
76 main.testSetUp.evnSetupConclusion( stepResult )
Jon Hall1efcb3f2016-08-23 13:42:15 -070077
Jon Hall1efcb3f2016-08-23 13:42:15 -070078 @staticmethod
79 def installOnos( main, vlanCfg=True ):
80 """
81 - Set up cell
82 - Create cell file
83 - Set cell file
84 - Verify cell file
85 - Kill ONOS process
86 - Uninstall ONOS cluster
87 - Verify ONOS start up
88 - Install ONOS cluster
89 - Connect to cli
90 """
91 # main.scale[ 0 ] determines the current number of ONOS controller
Jon Hall1efcb3f2016-08-23 13:42:15 -070092 if main.diff:
Devin Lim58046fa2017-07-05 16:55:00 -070093 main.apps = main.apps + "," + main.diff
Jon Hall1efcb3f2016-08-23 13:42:15 -070094 else:
95 main.log.error( "App list is empty" )
Devin Lim142b5342017-07-20 15:22:39 -070096 main.log.info( "NODE COUNT = " + str( main.Cluster.numCtrls ) )
97 main.log.info( ''.join( main.Cluster.getIps() ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -070098 main.dynamicHosts = [ 'in1', 'out1' ]
Devin Lim142b5342017-07-20 15:22:39 -070099 main.testSetUp.createApplyCell( main.Cluster, newCell=True, cellName=main.cellName,
100 Mininet=main.Mininet1, useSSH=Testcaselib.useSSH,
Devin Lim6301b742017-08-07 11:00:21 -0700101 ips=main.Cluster.getIps() )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700102 # kill off all onos processes
103 main.log.info( "Safety check, killing all ONOS processes" +
104 " before initiating environment setup" )
Devin Lim142b5342017-07-20 15:22:39 -0700105 for ctrl in main.Cluster.runningNodes:
106 main.ONOSbench.onosDie( ctrl.ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700107
Devin Lim142b5342017-07-20 15:22:39 -0700108 main.testSetUp.buildOnos( main.Cluster )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700109
Devin Lim142b5342017-07-20 15:22:39 -0700110 main.testSetUp.installOnos( main.Cluster, False )
Devin Lim58046fa2017-07-05 16:55:00 -0700111
Devin Lim142b5342017-07-20 15:22:39 -0700112 main.testSetUp.setupSsh( main.Cluster )
Devin Lim58046fa2017-07-05 16:55:00 -0700113
Devin Lim142b5342017-07-20 15:22:39 -0700114 main.testSetUp.checkOnosService( main.Cluster )
Devin Lim58046fa2017-07-05 16:55:00 -0700115
Devin Lim142b5342017-07-20 15:22:39 -0700116 cliResult = main.TRUE
Jon Hall1efcb3f2016-08-23 13:42:15 -0700117 main.step( "Checking if ONOS CLI is ready" )
Devin Lim142b5342017-07-20 15:22:39 -0700118 for ctrl in main.Cluster.runningNodes:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700119 ctrl.CLI.startCellCli()
Devin Lim142b5342017-07-20 15:22:39 -0700120 cliResult = cliResult and ctrl.CLI.startOnosCli( ctrl.ipAddress,
121 commandlineTimeout=60,
122 onosStartTimeout=100 )
123 ctrl.active = True
Jon Hall1efcb3f2016-08-23 13:42:15 -0700124 utilities.assert_equals( expect=main.TRUE,
125 actual=cliResult,
126 onpass="ONOS CLI is ready",
127 onfail="ONOS CLI is not ready" )
Devin Lim142b5342017-07-20 15:22:39 -0700128 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
129 main.FALSE,
130 sleep=10,
131 attempts=10 )
132 if ready:
133 ready = main.TRUE
134 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700135 onpass="ONOS summary command succeded",
136 onfail="ONOS summary command failed" )
137
138 with open( "%s/json/%s.json" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700139 main.dependencyPath, main.cfgName ) ) as cfg:
Devin Lim142b5342017-07-20 15:22:39 -0700140 main.Cluster.active( 0 ).REST.setNetCfg( json.load( cfg ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700141 with open( "%s/json/%s.chart" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700142 main.dependencyPath, main.cfgName ) ) as chart:
Jon Hall1efcb3f2016-08-23 13:42:15 -0700143 main.pingChart = json.load( chart )
144 if not ready:
145 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700146 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700147
Devin Lim142b5342017-07-20 15:22:39 -0700148 for ctrl in main.Cluster.active():
149 ctrl.CLI.logSet( "DEBUG", "org.onosproject.segmentrouting" )
150 ctrl.CLI.logSet( "DEBUG", "org.onosproject.driver.pipeline" )
151 ctrl.CLI.logSet( "DEBUG", "org.onosproject.store.group.impl" )
152 ctrl.CLI.logSet( "DEBUG", "org.onosproject.net.flowobjective.impl" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700153
154 @staticmethod
155 def startMininet( main, topology, args="" ):
156 main.step( "Starting Mininet Topology" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700157 arg = "--onos %d %s" % ( main.Cluster.numCtrls, args )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700158 main.topology = topology
159 topoResult = main.Mininet1.startNet(
160 topoFile=main.Mininet1.home + main.topology, args=arg )
161 stepResult = topoResult
162 utilities.assert_equals( expect=main.TRUE,
163 actual=stepResult,
164 onpass="Successfully loaded topology",
165 onfail="Failed to load topology" )
166 # Exit if topology did not load properly
167 if not topoResult:
Devin Lim44075962017-08-11 10:56:37 -0700168 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700169
170 @staticmethod
Devin Lim142b5342017-07-20 15:22:39 -0700171 def config( main, cfgName ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700172 main.spines = []
Piera2a7e1b2016-10-04 11:51:43 -0700173
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700174 main.failures = int( main.params[ 'failures' ] )
175 main.cfgName = cfgName
Piera2a7e1b2016-10-04 11:51:43 -0700176
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700177 if main.cfgName == '2x2':
178 spine = {}
179 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
180 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
181 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700182
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700183 spine = {}
184 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
185 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
186 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700187
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700188 elif main.cfgName == '4x4':
189 spine = {}
190 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
191 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
192 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700193
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700194 spine = {}
195 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
196 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
197 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700198
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700199 spine = {}
200 spine[ 'name' ] = main.params[ 'switches' ][ 'spine3' ]
201 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid3' ]
202 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700203
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700204 spine = {}
205 spine[ 'name' ] = main.params[ 'switches' ][ 'spine4' ]
206 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid4' ]
207 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700208
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700209 else:
Piera2a7e1b2016-10-04 11:51:43 -0700210 main.log.error( "Configuration failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700211 main.cleanAndExit()
Piera2a7e1b2016-10-04 11:51:43 -0700212
213 @staticmethod
Jon Hall1efcb3f2016-08-23 13:42:15 -0700214 def checkFlows( main, minFlowCount, dumpflows=True ):
215 main.step(
216 " Check whether the flow count is bigger than %s" % minFlowCount )
Devin Lim142b5342017-07-20 15:22:39 -0700217 count = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowCount,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700218 main.FALSE,
219 kwargs={ 'min': minFlowCount },
220 attempts=10,
221 sleep=10 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700222 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700223 expect=True,
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700224 actual=( count > 0 ),
Jon Hall1efcb3f2016-08-23 13:42:15 -0700225 onpass="Flow count looks correct: " + str( count ),
226 onfail="Flow count looks wrong: " + str( count ) )
227
228 main.step( "Check whether all flow status are ADDED" )
Devin Lim142b5342017-07-20 15:22:39 -0700229 flowCheck = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowsState,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700230 main.FALSE,
231 kwargs={ 'isPENDING': False },
232 attempts=2,
233 sleep=10 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700234 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700235 expect=main.TRUE,
236 actual=flowCheck,
237 onpass="Flow status is correct!",
238 onfail="Flow status is wrong!" )
239 if dumpflows:
Devin Lim142b5342017-07-20 15:22:39 -0700240 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700241 "flows",
242 main.logdir,
243 "flowsBefore" + main.cfgName )
Devin Lim142b5342017-07-20 15:22:39 -0700244 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700245 "groups",
246 main.logdir,
247 "groupsBefore" + main.cfgName )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700248
249 @staticmethod
250 def pingAll( main, tag="", dumpflows=True ):
251 main.log.report( "Check full connectivity" )
252 print main.pingChart
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700253 for entry in main.pingChart.itervalues():
Jon Hall1efcb3f2016-08-23 13:42:15 -0700254 print entry
255 hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
256 expect = main.TRUE if expect else main.FALSE
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700257 main.step( "Connectivity for %s %s" % ( str( hosts ), tag ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700258 pa = main.Mininet1.pingallHosts( hosts )
259 utilities.assert_equals( expect=expect, actual=pa,
260 onpass="IP connectivity successfully tested",
261 onfail="IP connectivity failed" )
262 if dumpflows:
Devin Lim142b5342017-07-20 15:22:39 -0700263 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700264 "flows",
265 main.logdir,
266 "flowsOn" + tag )
Devin Lim142b5342017-07-20 15:22:39 -0700267 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700268 "groups",
269 main.logdir,
270 "groupsOn" + tag )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700271
272 @staticmethod
273 def killLink( main, end1, end2, switches, links ):
274 """
275 end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
276 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
277 Kill a link and verify ONOS can see the proper link change
278 """
279 main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700280 main.step( "Kill link between %s and %s" % ( end1, end2 ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700281 LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
Pierfb719b12016-09-19 14:51:44 -0700282 LinkDown = main.Mininet1.link( END2=end1, END1=end2, OPTION="down" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700283 main.log.info(
284 "Waiting %s seconds for link down to be discovered" % main.linkSleep )
285 time.sleep( main.linkSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700286 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700287 main.FALSE,
288 kwargs={ 'numoswitch': switches,
289 'numolink': links },
290 attempts=10,
291 sleep=main.linkSleep )
292 result = topology & LinkDown
293 utilities.assert_equals( expect=main.TRUE, actual=result,
294 onpass="Link down successful",
295 onfail="Failed to turn off link?" )
296
297 @staticmethod
298 def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
299 links ):
300 """
301 Params:
302 end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
303 dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
304 port1, port2: respective port of the end switches that connects to the link, ex.:'1'
305 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
306 Kill a link and verify ONOS can see the proper link change
307 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700308 main.step( "Restore link between %s and %s" % ( end1, end2 ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700309 result = False
310 count = 0
311 while True:
312 count += 1
313 main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
314 main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
315 main.log.info(
316 "Waiting %s seconds for link up to be discovered" % main.linkSleep )
317 time.sleep( main.linkSleep )
Pierfb719b12016-09-19 14:51:44 -0700318
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700319 for i in range( 0, main.Cluster.numCtrls ):
Devin Lim142b5342017-07-20 15:22:39 -0700320 ctrl = main.Cluster.runningNodes[ i ]
321 onosIsUp = main.ONOSbench.isup( ctrl.ipAddress )
Pierfb719b12016-09-19 14:51:44 -0700322 if onosIsUp == main.TRUE:
Devin Lim142b5342017-07-20 15:22:39 -0700323 ctrl.CLI.portstate( dpid=dpid1, port=port1 )
324 ctrl.CLI.portstate( dpid=dpid2, port=port2 )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700325 time.sleep( main.linkSleep )
326
Devin Lim142b5342017-07-20 15:22:39 -0700327 result = main.Cluster.active( 0 ).CLI.checkStatus( numoswitch=switches,
328 numolink=links )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700329 if count > 5 or result:
330 break
331 utilities.assert_equals( expect=main.TRUE, actual=result,
332 onpass="Link up successful",
333 onfail="Failed to bring link up" )
334
335 @staticmethod
336 def killSwitch( main, switch, switches, links ):
337 """
338 Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
339 Completely kill a switch and verify ONOS can see the proper change
340 """
341 main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
342 main.step( "Kill " + switch )
343 main.log.info( "Stopping" + switch )
344 main.Mininet1.switch( SW=switch, OPTION="stop" )
345 # todo make this repeatable
346 main.log.info( "Waiting %s seconds for switch down to be discovered" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700347 main.switchSleep ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700348 time.sleep( main.switchSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700349 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700350 main.FALSE,
351 kwargs={ 'numoswitch': switches,
352 'numolink': links },
353 attempts=10,
354 sleep=main.switchSleep )
355 utilities.assert_equals( expect=main.TRUE, actual=topology,
356 onpass="Kill switch successful",
357 onfail="Failed to kill switch?" )
358
359 @staticmethod
360 def recoverSwitch( main, switch, switches, links ):
361 """
362 Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
363 Recover a switch and verify ONOS can see the proper change
364 """
365 # todo make this repeatable
366 main.step( "Recovering " + switch )
367 main.log.info( "Starting" + switch )
368 main.Mininet1.switch( SW=switch, OPTION="start" )
369 main.log.info( "Waiting %s seconds for switch up to be discovered" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700370 main.switchSleep ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700371 time.sleep( main.switchSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700372 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700373 main.FALSE,
374 kwargs={ 'numoswitch': switches,
375 'numolink': links },
376 attempts=10,
377 sleep=main.switchSleep )
378 utilities.assert_equals( expect=main.TRUE, actual=topology,
379 onpass="Switch recovery successful",
380 onfail="Failed to recover switch?" )
381
382 @staticmethod
383 def cleanup( main ):
384 """
385 Stop Onos-cluster.
386 Stops Mininet
387 Copies ONOS log
388 """
Devin Lim58046fa2017-07-05 16:55:00 -0700389 try:
390 from tests.dependencies.utils import Utils
391 except ImportError:
392 main.log.error( "Utils not found exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -0700393 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -0700394 try:
Devin Lim142b5342017-07-20 15:22:39 -0700395 main.utils
Devin Lim58046fa2017-07-05 16:55:00 -0700396 except ( NameError, AttributeError ):
Devin Lim142b5342017-07-20 15:22:39 -0700397 main.utils = Utils()
Devin Lim58046fa2017-07-05 16:55:00 -0700398
399 main.utils.mininetCleanup( main.Mininet1 )
400
Devin Lim142b5342017-07-20 15:22:39 -0700401 main.utils.copyKarafLog( main.cfgName )
Devin Lim58046fa2017-07-05 16:55:00 -0700402
Devin Lim142b5342017-07-20 15:22:39 -0700403 for ctrl in main.Cluster.active():
404 main.ONOSbench.onosStop( ctrl.ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700405
406 @staticmethod
407 def killOnos( main, nodes, switches, links, expNodes ):
408 """
409 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
410 switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
411 Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
412 """
413 main.step( "Killing ONOS instance" )
Pier3b58c652016-09-26 12:03:31 -0700414
Jon Hall1efcb3f2016-08-23 13:42:15 -0700415 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700416 killResult = main.ONOSbench.onosDie( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700417 utilities.assert_equals( expect=main.TRUE, actual=killResult,
418 onpass="ONOS instance Killed",
419 onfail="Error killing ONOS instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700420 main.Cluster.runningNodes[ i ].active = False
Jon Hall1efcb3f2016-08-23 13:42:15 -0700421 time.sleep( 12 )
Pier3b58c652016-09-26 12:03:31 -0700422
Devin Lim142b5342017-07-20 15:22:39 -0700423 if len( nodes ) < main.Cluster.numCtrls:
Pier3b58c652016-09-26 12:03:31 -0700424
Pier3b58c652016-09-26 12:03:31 -0700425 nodeResults = utilities.retry( Testcaselib.nodesCheck,
426 False,
Pier3b58c652016-09-26 12:03:31 -0700427 attempts=5,
428 sleep=10 )
429 utilities.assert_equals( expect=True, actual=nodeResults,
430 onpass="Nodes check successful",
431 onfail="Nodes check NOT successful" )
432
433 if not nodeResults:
434 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700435 ctrl = main.Cluster.runningNodes[ i ]
Pier3b58c652016-09-26 12:03:31 -0700436 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700437 ctrl.name,
438 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Pier3b58c652016-09-26 12:03:31 -0700439 main.log.error( "Failed to kill ONOS, stopping test" )
Devin Lim44075962017-08-11 10:56:37 -0700440 main.cleanAndExit()
Pier3b58c652016-09-26 12:03:31 -0700441
Devin Lim142b5342017-07-20 15:22:39 -0700442 topology = utilities.retry( main.Cluster.active( 0 ).checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700443 main.FALSE,
444 kwargs={ 'numoswitch': switches,
445 'numolink': links,
446 'numoctrl': expNodes },
447 attempts=10,
448 sleep=12 )
449 utilities.assert_equals( expect=main.TRUE, actual=topology,
450 onpass="ONOS Instance down successful",
451 onfail="Failed to turn off ONOS Instance" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700452
453 @staticmethod
454 def recoverOnos( main, nodes, switches, links, expNodes ):
455 """
456 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
457 switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
458 Recover an ONOS instance and verify the ONOS cluster can see the proper change
459 """
460 main.step( "Recovering ONOS instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700461 [ main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) for i in nodes ]
Jon Hall1efcb3f2016-08-23 13:42:15 -0700462 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700463 isUp = main.ONOSbench.isup( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700464 utilities.assert_equals( expect=main.TRUE, actual=isUp,
465 onpass="ONOS service is ready",
466 onfail="ONOS service did not start properly" )
467 for i in nodes:
468 main.step( "Checking if ONOS CLI is ready" )
Devin Lim142b5342017-07-20 15:22:39 -0700469 ctrl = main.Cluster.runningNodes[ i ]
470 ctrl.CLI.startCellCli()
471 cliResult = ctrl.CLI.startOnosCli( ctrl.ipAddress,
472 commandlineTimeout=60,
473 onosStartTimeout=100 )
474 ctrl.active = True
Jon Hall1efcb3f2016-08-23 13:42:15 -0700475 utilities.assert_equals( expect=main.TRUE,
476 actual=cliResult,
477 onpass="ONOS CLI is ready",
478 onfail="ONOS CLI is not ready" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700479
Pier3b58c652016-09-26 12:03:31 -0700480 main.step( "Checking ONOS nodes" )
481 nodeResults = utilities.retry( Testcaselib.nodesCheck,
482 False,
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700483 args=[ nodes ],
Pier3b58c652016-09-26 12:03:31 -0700484 attempts=5,
485 sleep=10 )
486 utilities.assert_equals( expect=True, actual=nodeResults,
487 onpass="Nodes check successful",
488 onfail="Nodes check NOT successful" )
489
490 if not nodeResults:
491 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700492 ctrl = main.Cluster.runningNodes[ i ]
Pier3b58c652016-09-26 12:03:31 -0700493 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700494 ctrl.name,
495 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Pier3b58c652016-09-26 12:03:31 -0700496 main.log.error( "Failed to start ONOS, stopping test" )
Devin Lim44075962017-08-11 10:56:37 -0700497 main.cleanAndExit()
Pier3b58c652016-09-26 12:03:31 -0700498
Devin Lim142b5342017-07-20 15:22:39 -0700499 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700500 main.FALSE,
501 kwargs={ 'numoswitch': switches,
502 'numolink': links,
503 'numoctrl': expNodes },
504 attempts=10,
505 sleep=12 )
506 utilities.assert_equals( expect=main.TRUE, actual=topology,
507 onpass="ONOS Instance down successful",
508 onfail="Failed to turn off ONOS Instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700509 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
510 main.FALSE,
511 attempts=10,
512 sleep=12 )
513 if ready:
514 ready = main.TRUE
515 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700516 onpass="ONOS summary command succeded",
517 onfail="ONOS summary command failed" )
518 if not ready:
519 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700520 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700521
522 @staticmethod
523 def addHostCfg( main ):
524 """
525 Adds Host Configuration to ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700526 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700527 """
528 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700529 hostCfg = {}
Jon Hall1efcb3f2016-08-23 13:42:15 -0700530 with open( main.dependencyPath + "/json/extra.json" ) as template:
531 hostCfg = json.load( template )
532 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'in1' ]
533 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700534 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700535 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
536 subjectClass="hosts",
537 subjectKey=urllib.quote( mac,
538 safe='' ),
539 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700540 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'out1' ]
541 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700542 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700543 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
544 subjectClass="hosts",
545 subjectKey=urllib.quote( mac,
546 safe='' ),
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700547 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700548 main.pingChart.update( { 'vlan1': { "expect": "True",
549 "hosts": [ "olt1", "vsg1" ] } } )
550 main.pingChart[ 'vlan5' ][ 'expect' ] = 0
551 main.pingChart[ 'vlan10' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700552 ports = "[%s,%s]" % ( 5, 6 )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700553 cfg = '{"of:0000000000000001":[{"vlan":1,"ports":%s,"name":"OLT 1"}]}' % ports
Devin Lim142b5342017-07-20 15:22:39 -0700554 main.Cluster.active( 0 ).REST.setNetCfg( json.loads( cfg ),
555 subjectClass="apps",
556 subjectKey="org.onosproject.segmentrouting",
557 configKey="xconnect" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700558
559 @staticmethod
560 def delHostCfg( main ):
561 """
562 Removest Host Configuration from ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700563 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700564 """
565 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700566 hostCfg = {}
Jon Hall1efcb3f2016-08-23 13:42:15 -0700567 with open( main.dependencyPath + "/json/extra.json" ) as template:
568 hostCfg = json.load( template )
569 main.step( "Removing host configuration" )
570 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700571 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700572 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
573 subjectKey=urllib.quote(
574 mac,
575 safe='' ),
576 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700577 main.step( "Removing configuration" )
578 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700579 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700580 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
581 subjectKey=urllib.quote(
582 mac,
583 safe='' ),
584 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700585 main.step( "Removing vlan configuration" )
586 main.pingChart[ 'vlan1' ][ 'expect' ] = 0
Devin Lim142b5342017-07-20 15:22:39 -0700587 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="apps",
588 subjectKey="org.onosproject.segmentrouting",
589 configKey="xconnect" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700590
Pier3b58c652016-09-26 12:03:31 -0700591 @staticmethod
592 def nodesCheck( nodes ):
Pier3b58c652016-09-26 12:03:31 -0700593 results = True
Devin Lim142b5342017-07-20 15:22:39 -0700594 nodesOutput = main.Cluster.command( "nodes", specificDriver=2 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700595 ips = sorted( main.Cluster.getIps( activeOnly=True ) )
Pier3b58c652016-09-26 12:03:31 -0700596 for i in nodesOutput:
597 try:
598 current = json.loads( i )
599 activeIps = []
600 currentResult = False
601 for node in current:
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700602 if node[ 'state' ] == 'READY':
603 activeIps.append( node[ 'ip' ] )
Pier3b58c652016-09-26 12:03:31 -0700604 currentResult = True
605 for ip in ips:
606 if ip not in activeIps:
607 currentResult = False
608 break
609 except ( ValueError, TypeError ):
610 main.log.error( "Error parsing nodes output" )
611 main.log.warn( repr( i ) )
612 currentResult = False
613 results = results and currentResult
614 return results