blob: 8d929adc87881d27a96989d59aa9ab9ab929e974 [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2016 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jon Hall1efcb3f2016-08-23 13:42:15 -070021import os
22import imp
23import time
24import json
25import urllib
26from core import utilities
27
28
29class Testcaselib:
Pierfb719b12016-09-19 14:51:44 -070030
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070031 useSSH = True
Pierfb719b12016-09-19 14:51:44 -070032
Jon Hall1efcb3f2016-08-23 13:42:15 -070033 @staticmethod
34 def initTest( main ):
35 """
36 - Construct tests variables
37 - GIT ( optional )
38 - Checkout ONOS master branch
39 - Pull latest ONOS code
40 - Building ONOS ( optional )
41 - Install ONOS package
42 - Build ONOS package
43 """
Devin Lim58046fa2017-07-05 16:55:00 -070044 try:
45 from tests.dependencies.ONOSSetup import ONOSSetup
46 main.testSetUp = ONOSSetup()
47 except ImportError:
48 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070049 main.cleanAndExit()
You Wangd5873482018-01-24 12:30:00 -080050 from tests.dependencies.Network import Network
51 main.Network = Network()
Devin Lim0c972b72018-02-08 14:53:59 -080052 main.testSetUp.envSetupDescription( False )
Devin Lim58046fa2017-07-05 16:55:00 -070053 stepResult = main.FALSE
54 try:
55 main.step( "Constructing test variables" )
56 # Test variables
57 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
58 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
Devin Lim58046fa2017-07-05 16:55:00 -070059 main.path = os.path.dirname( main.testFile )
You Wangac02b142018-01-26 14:57:28 -080060 main.topoPath = main.path + "/../dependencies/"
61 main.configPath = main.path + "/../dependencies/"
Devin Lim58046fa2017-07-05 16:55:00 -070062 main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
You Wangd87b2312018-01-30 12:47:17 -080063 main.topologyLib = main.params[ 'DEPENDENCY' ][ 'lib' ] if 'lib' in main.params[ 'DEPENDENCY' ] else None
64 main.topologyConf = main.params[ 'DEPENDENCY' ][ 'conf' ] if 'conf' in main.params[ 'DEPENDENCY' ] else None
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070065 main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
Devin Lim58046fa2017-07-05 16:55:00 -070066 main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
Devin Lim58046fa2017-07-05 16:55:00 -070067 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
Jon Hall1efcb3f2016-08-23 13:42:15 -070068
Devin Lim0c972b72018-02-08 14:53:59 -080069 stepResult = main.testSetUp.envSetup( False )
Devin Lim58046fa2017-07-05 16:55:00 -070070 except Exception as e:
71 main.testSetUp.envSetupException( e )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -070072
Devin Lim58046fa2017-07-05 16:55:00 -070073 main.testSetUp.evnSetupConclusion( stepResult )
Jon Hall1efcb3f2016-08-23 13:42:15 -070074
Jon Hall1efcb3f2016-08-23 13:42:15 -070075 @staticmethod
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -080076 def installOnos( main, vlanCfg=True, skipPackage=False, cliSleep=10,
77 parallel=True ):
Jon Hall1efcb3f2016-08-23 13:42:15 -070078 """
79 - Set up cell
80 - Create cell file
81 - Set cell file
82 - Verify cell file
83 - Kill ONOS process
84 - Uninstall ONOS cluster
85 - Verify ONOS start up
86 - Install ONOS cluster
87 - Connect to cli
88 """
89 # main.scale[ 0 ] determines the current number of ONOS controller
You Wangd87b2312018-01-30 12:47:17 -080090 if not main.apps:
Jon Hall1efcb3f2016-08-23 13:42:15 -070091 main.log.error( "App list is empty" )
Devin Lim142b5342017-07-20 15:22:39 -070092 main.log.info( "NODE COUNT = " + str( main.Cluster.numCtrls ) )
93 main.log.info( ''.join( main.Cluster.getIps() ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -070094 main.dynamicHosts = [ 'in1', 'out1' ]
You Wanga0f6ff62018-01-11 15:46:30 -080095 main.testSetUp.ONOSSetUp( main.Cluster, newCell=True, cellName=main.cellName,
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -080096 skipPack=skipPackage,
97 useSSH=Testcaselib.useSSH,
Devin Lim0c972b72018-02-08 14:53:59 -080098 installParallel=parallel, includeCaseDesc=False )
Devin Lim142b5342017-07-20 15:22:39 -070099 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
100 main.FALSE,
You Wang1cdc5f52017-12-19 16:47:51 -0800101 sleep=cliSleep,
Devin Lim142b5342017-07-20 15:22:39 -0700102 attempts=10 )
103 if ready:
104 ready = main.TRUE
105 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700106 onpass="ONOS summary command succeded",
107 onfail="ONOS summary command failed" )
108
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -0800109 with open( "%s/json/%s.json" % (main.configPath, main.cfgName)) as cfg:
110 main.Cluster.active( 0 ).REST.setNetCfg(json.load(cfg))
111 with open("%s/json/%s.chart" % (main.configPath, main.cfgName)) as chart:
112 main.pingChart = json.load(chart)
Jon Hall1efcb3f2016-08-23 13:42:15 -0700113 if not ready:
114 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700115 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700116
Devin Lim142b5342017-07-20 15:22:39 -0700117 for ctrl in main.Cluster.active():
118 ctrl.CLI.logSet( "DEBUG", "org.onosproject.segmentrouting" )
119 ctrl.CLI.logSet( "DEBUG", "org.onosproject.driver.pipeline" )
120 ctrl.CLI.logSet( "DEBUG", "org.onosproject.store.group.impl" )
121 ctrl.CLI.logSet( "DEBUG", "org.onosproject.net.flowobjective.impl" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700122
123 @staticmethod
124 def startMininet( main, topology, args="" ):
You Wangd87b2312018-01-30 12:47:17 -0800125 copyResult = main.ONOSbench.scp( main.Mininet1,
126 main.topoPath + main.topology,
127 main.Mininet1.home,
128 direction="to" )
129 if main.topologyLib:
130 for lib in main.topologyLib.split(","):
131 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
132 main.topoPath + lib,
133 main.Mininet1.home,
134 direction="to" )
135 if main.topologyConf:
136 for conf in main.topologyConf.split(","):
137 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
138 main.topoPath + "conf/" + conf,
139 "~/",
140 direction="to" )
141 stepResult = copyResult
142 utilities.assert_equals( expect=main.TRUE,
143 actual=stepResult,
144 onpass="Successfully copied topo files",
145 onfail="Failed to copy topo files" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700146 main.step( "Starting Mininet Topology" )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700147 arg = "--onos-ip=%s %s" % (",".join([ctrl.ipAddress for ctrl in main.Cluster.runningNodes]), args)
Jon Hall1efcb3f2016-08-23 13:42:15 -0700148 main.topology = topology
149 topoResult = main.Mininet1.startNet(
150 topoFile=main.Mininet1.home + main.topology, args=arg )
151 stepResult = topoResult
152 utilities.assert_equals( expect=main.TRUE,
153 actual=stepResult,
154 onpass="Successfully loaded topology",
155 onfail="Failed to load topology" )
156 # Exit if topology did not load properly
157 if not topoResult:
Devin Lim44075962017-08-11 10:56:37 -0700158 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700159
160 @staticmethod
You Wang84f981d2018-01-12 16:11:50 -0800161 def connectToPhysicalNetwork( main, switchNames ):
162 main.step( "Connecting to physical netowrk" )
163 topoResult = main.NetworkBench.connectToNet()
164 stepResult = topoResult
165 utilities.assert_equals( expect=main.TRUE,
166 actual=stepResult,
167 onpass="Successfully loaded topology",
168 onfail="Failed to load topology" )
169 # Exit if topology did not load properly
170 if not topoResult:
171 main.cleanAndExit()
172
173 main.step( "Assign switches to controllers." )
174 assignResult = main.TRUE
175 for name in switchNames:
176 assignResult = assignResult & main.NetworkBench.assignSwController( sw=name,
177 ip=main.Cluster.getIps(),
178 port='6653' )
179 utilities.assert_equals( expect=main.TRUE,
180 actual=stepResult,
181 onpass="Successfully assign switches to controllers",
182 onfail="Failed to assign switches to controllers" )
183
184 @staticmethod
Devin Lim142b5342017-07-20 15:22:39 -0700185 def config( main, cfgName ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700186 main.spines = []
Piera2a7e1b2016-10-04 11:51:43 -0700187
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700188 main.failures = int( main.params[ 'failures' ] )
189 main.cfgName = cfgName
Piera2a7e1b2016-10-04 11:51:43 -0700190
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700191 if main.cfgName == '2x2':
192 spine = {}
193 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
194 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
195 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700196
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700197 spine = {}
198 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
199 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
200 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700201
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700202 elif main.cfgName == '4x4':
203 spine = {}
204 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
205 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
206 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700207
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700208 spine = {}
209 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
210 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
211 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700212
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700213 spine = {}
214 spine[ 'name' ] = main.params[ 'switches' ][ 'spine3' ]
215 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid3' ]
216 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700217
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700218 spine = {}
219 spine[ 'name' ] = main.params[ 'switches' ][ 'spine4' ]
220 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid4' ]
221 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700222
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700223 else:
Piera2a7e1b2016-10-04 11:51:43 -0700224 main.log.error( "Configuration failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700225 main.cleanAndExit()
Piera2a7e1b2016-10-04 11:51:43 -0700226
227 @staticmethod
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900228 def checkFlows( main, minFlowCount, tag="", dumpflows=True, sleep=10 ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700229 main.step(
230 " Check whether the flow count is bigger than %s" % minFlowCount )
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900231 if tag == "":
232 tag = 'CASE%d' % main.CurrentTestCaseNumber
Devin Lim142b5342017-07-20 15:22:39 -0700233 count = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowCount,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700234 main.FALSE,
235 kwargs={ 'min': minFlowCount },
236 attempts=10,
You Wang1cdc5f52017-12-19 16:47:51 -0800237 sleep=sleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700238 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700239 expect=True,
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700240 actual=( count > 0 ),
Jon Hall1efcb3f2016-08-23 13:42:15 -0700241 onpass="Flow count looks correct: " + str( count ),
242 onfail="Flow count looks wrong: " + str( count ) )
243
244 main.step( "Check whether all flow status are ADDED" )
Devin Lim142b5342017-07-20 15:22:39 -0700245 flowCheck = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowsState,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700246 main.FALSE,
247 kwargs={ 'isPENDING': False },
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800248 attempts=5,
You Wang1cdc5f52017-12-19 16:47:51 -0800249 sleep=sleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700250 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700251 expect=main.TRUE,
252 actual=flowCheck,
253 onpass="Flow status is correct!",
254 onfail="Flow status is wrong!" )
255 if dumpflows:
Devin Lim142b5342017-07-20 15:22:39 -0700256 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700257 "flows",
258 main.logdir,
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900259 tag + "_FlowsBefore" )
Devin Lim142b5342017-07-20 15:22:39 -0700260 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700261 "groups",
262 main.logdir,
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900263 tag + "_GroupsBefore" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700264
265 @staticmethod
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800266 def checkFlowsByDpid( main, dpid, minFlowCount, sleep=10 ):
267 main.step(
268 " Check whether the flow count of device %s is bigger than %s" % ( dpid, minFlowCount ) )
269 count = utilities.retry( main.Cluster.active( 0 ).CLI.flowAddedCount,
270 None,
271 args=( dpid, ),
272 attempts=5,
273 sleep=sleep )
274 utilities.assertEquals(
275 expect=True,
276 actual=( int( count ) > minFlowCount ),
277 onpass="Flow count looks correct: " + count ,
278 onfail="Flow count looks wrong: " + count )
279
280 @staticmethod
Andreas Pantelopoulosf6ed5012018-02-08 21:26:01 -0800281 def pingAllBasedOnIp( main, tag="", dumpflows=True ):
282 main.log.report( "Check full connectivity" )
283 print main.pingChart
284 if tag == "":
285 tag = 'CASE%d' % main.CurrentTestCaseNumber
286 for entry in main.pingChart.itervalues():
287 print entry
288 hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
289 try:
290 expect = main.TRUE if str(expect).lower() == 'true' else main.FALSE
291 except:
292 expect = main.FALSE
293 main.step( "Connectivity for %s %s" % ( str( hosts ), tag ) )
294
295 if ("v4" in hosts[0]):
296 pa = main.Network.pingallHosts( hosts )
297 utilities.assert_equals( expect=expect, actual=pa,
298 onpass="IPv4 connectivity successfully tested",
299 onfail="IPv4 connectivity failed" )
300 if ("v6" in hosts[0]):
301 pa = main.Network.pingIpv6Hosts( hosts )
302 utilities.assert_equals( expect=expect, actual=pa,
303 onpass="IPv6 connectivity successfully tested",
304 onfail="IPv6 connectivity failed" )
305
306 if dumpflows:
307 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
308 "flows",
309 main.logdir,
310 tag + "_FlowsOn" )
311 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
312 "groups",
313 main.logdir,
314 tag + "_GroupsOn" )
315
316 @staticmethod
Jon Hall1efcb3f2016-08-23 13:42:15 -0700317 def pingAll( main, tag="", dumpflows=True ):
318 main.log.report( "Check full connectivity" )
319 print main.pingChart
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900320 if tag == "":
321 tag = 'CASE%d' % main.CurrentTestCaseNumber
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700322 for entry in main.pingChart.itervalues():
Jon Hall1efcb3f2016-08-23 13:42:15 -0700323 print entry
324 hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700325 try:
326 expect = main.TRUE if str(expect).lower() == 'true' else main.FALSE
327 except:
328 expect = main.FALSE
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700329 main.step( "Connectivity for %s %s" % ( str( hosts ), tag ) )
You Wangd5873482018-01-24 12:30:00 -0800330 pa = main.Network.pingallHosts( hosts )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700331
Jon Hall1efcb3f2016-08-23 13:42:15 -0700332 utilities.assert_equals( expect=expect, actual=pa,
333 onpass="IP connectivity successfully tested",
334 onfail="IP connectivity failed" )
335 if dumpflows:
Devin Lim142b5342017-07-20 15:22:39 -0700336 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700337 "flows",
338 main.logdir,
Devin Lim97b6b862018-01-23 22:51:25 -0800339 tag + "_FlowsOn" )
Devin Lim142b5342017-07-20 15:22:39 -0700340 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700341 "groups",
342 main.logdir,
Devin Lim97b6b862018-01-23 22:51:25 -0800343 tag + "_GroupsOn" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700344
345 @staticmethod
346 def killLink( main, end1, end2, switches, links ):
347 """
348 end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
349 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
350 Kill a link and verify ONOS can see the proper link change
351 """
352 main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700353 main.step( "Kill link between %s and %s" % ( end1, end2 ) )
You Wangd5873482018-01-24 12:30:00 -0800354 LinkDown = main.Network.link( END1=end1, END2=end2, OPTION="down" )
355 LinkDown = main.Network.link( END2=end1, END1=end2, OPTION="down" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700356 main.log.info(
357 "Waiting %s seconds for link down to be discovered" % main.linkSleep )
358 time.sleep( main.linkSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700359 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700360 main.FALSE,
361 kwargs={ 'numoswitch': switches,
362 'numolink': links },
363 attempts=10,
364 sleep=main.linkSleep )
365 result = topology & LinkDown
366 utilities.assert_equals( expect=main.TRUE, actual=result,
367 onpass="Link down successful",
368 onfail="Failed to turn off link?" )
369
370 @staticmethod
371 def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
372 links ):
373 """
374 Params:
375 end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
376 dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
377 port1, port2: respective port of the end switches that connects to the link, ex.:'1'
378 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
379 Kill a link and verify ONOS can see the proper link change
380 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700381 main.step( "Restore link between %s and %s" % ( end1, end2 ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700382 result = False
383 count = 0
384 while True:
385 count += 1
You Wangd5873482018-01-24 12:30:00 -0800386 main.Network.link( END1=end1, END2=end2, OPTION="up" )
387 main.Network.link( END2=end1, END1=end2, OPTION="up" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700388 main.log.info(
389 "Waiting %s seconds for link up to be discovered" % main.linkSleep )
390 time.sleep( main.linkSleep )
Pierfb719b12016-09-19 14:51:44 -0700391
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700392 for i in range( 0, main.Cluster.numCtrls ):
Devin Lim142b5342017-07-20 15:22:39 -0700393 ctrl = main.Cluster.runningNodes[ i ]
394 onosIsUp = main.ONOSbench.isup( ctrl.ipAddress )
Pierfb719b12016-09-19 14:51:44 -0700395 if onosIsUp == main.TRUE:
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900396 ctrl.CLI.portstate( dpid=dpid1, port=port1, state='Enable' )
397 ctrl.CLI.portstate( dpid=dpid2, port=port2, state='Enable' )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700398 time.sleep( main.linkSleep )
399
Devin Lim142b5342017-07-20 15:22:39 -0700400 result = main.Cluster.active( 0 ).CLI.checkStatus( numoswitch=switches,
401 numolink=links )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700402 if count > 5 or result:
403 break
404 utilities.assert_equals( expect=main.TRUE, actual=result,
405 onpass="Link up successful",
406 onfail="Failed to bring link up" )
407
408 @staticmethod
409 def killSwitch( main, switch, switches, links ):
410 """
411 Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
412 Completely kill a switch and verify ONOS can see the proper change
413 """
414 main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
415 main.step( "Kill " + switch )
416 main.log.info( "Stopping" + switch )
You Wangd5873482018-01-24 12:30:00 -0800417 main.Network.switch( SW=switch, OPTION="stop" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700418 # todo make this repeatable
419 main.log.info( "Waiting %s seconds for switch down to be discovered" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700420 main.switchSleep ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700421 time.sleep( main.switchSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700422 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700423 main.FALSE,
424 kwargs={ 'numoswitch': switches,
425 'numolink': links },
426 attempts=10,
427 sleep=main.switchSleep )
428 utilities.assert_equals( expect=main.TRUE, actual=topology,
429 onpass="Kill switch successful",
430 onfail="Failed to kill switch?" )
431
432 @staticmethod
433 def recoverSwitch( main, switch, switches, links ):
434 """
435 Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
436 Recover a switch and verify ONOS can see the proper change
437 """
438 # todo make this repeatable
439 main.step( "Recovering " + switch )
440 main.log.info( "Starting" + switch )
You Wangd5873482018-01-24 12:30:00 -0800441 main.Network.switch( SW=switch, OPTION="start" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700442 main.log.info( "Waiting %s seconds for switch up to be discovered" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700443 main.switchSleep ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700444 time.sleep( main.switchSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700445 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700446 main.FALSE,
447 kwargs={ 'numoswitch': switches,
448 'numolink': links },
449 attempts=10,
450 sleep=main.switchSleep )
451 utilities.assert_equals( expect=main.TRUE, actual=topology,
452 onpass="Switch recovery successful",
453 onfail="Failed to recover switch?" )
454
455 @staticmethod
456 def cleanup( main ):
457 """
458 Stop Onos-cluster.
459 Stops Mininet
460 Copies ONOS log
461 """
Devin Lim58046fa2017-07-05 16:55:00 -0700462 try:
463 from tests.dependencies.utils import Utils
464 except ImportError:
465 main.log.error( "Utils not found exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -0700466 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -0700467 try:
Devin Lim142b5342017-07-20 15:22:39 -0700468 main.utils
Devin Lim58046fa2017-07-05 16:55:00 -0700469 except ( NameError, AttributeError ):
Devin Lim142b5342017-07-20 15:22:39 -0700470 main.utils = Utils()
Devin Lim58046fa2017-07-05 16:55:00 -0700471
472 main.utils.mininetCleanup( main.Mininet1 )
473
Devin Lim0c972b72018-02-08 14:53:59 -0800474 main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True, includeCaseDesc=False )
Devin Lim58046fa2017-07-05 16:55:00 -0700475
Devin Lim142b5342017-07-20 15:22:39 -0700476 for ctrl in main.Cluster.active():
477 main.ONOSbench.onosStop( ctrl.ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700478
479 @staticmethod
480 def killOnos( main, nodes, switches, links, expNodes ):
481 """
482 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
483 switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
484 Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
485 """
486 main.step( "Killing ONOS instance" )
Pier3b58c652016-09-26 12:03:31 -0700487
Jon Hall1efcb3f2016-08-23 13:42:15 -0700488 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700489 killResult = main.ONOSbench.onosDie( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700490 utilities.assert_equals( expect=main.TRUE, actual=killResult,
491 onpass="ONOS instance Killed",
492 onfail="Error killing ONOS instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700493 main.Cluster.runningNodes[ i ].active = False
Jon Hall1efcb3f2016-08-23 13:42:15 -0700494 time.sleep( 12 )
Pier3b58c652016-09-26 12:03:31 -0700495
Devin Lim142b5342017-07-20 15:22:39 -0700496 if len( nodes ) < main.Cluster.numCtrls:
Pier3b58c652016-09-26 12:03:31 -0700497
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700498 nodeResults = utilities.retry( main.Cluster.nodesCheck,
Pier3b58c652016-09-26 12:03:31 -0700499 False,
Pier3b58c652016-09-26 12:03:31 -0700500 attempts=5,
501 sleep=10 )
502 utilities.assert_equals( expect=True, actual=nodeResults,
503 onpass="Nodes check successful",
504 onfail="Nodes check NOT successful" )
505
506 if not nodeResults:
507 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700508 ctrl = main.Cluster.runningNodes[ i ]
Pier3b58c652016-09-26 12:03:31 -0700509 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700510 ctrl.name,
511 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Pier3b58c652016-09-26 12:03:31 -0700512 main.log.error( "Failed to kill ONOS, stopping test" )
Devin Lim44075962017-08-11 10:56:37 -0700513 main.cleanAndExit()
Pier3b58c652016-09-26 12:03:31 -0700514
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900515 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700516 main.FALSE,
517 kwargs={ 'numoswitch': switches,
518 'numolink': links,
519 'numoctrl': expNodes },
520 attempts=10,
521 sleep=12 )
522 utilities.assert_equals( expect=main.TRUE, actual=topology,
523 onpass="ONOS Instance down successful",
524 onfail="Failed to turn off ONOS Instance" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700525
526 @staticmethod
527 def recoverOnos( main, nodes, switches, links, expNodes ):
528 """
529 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
530 switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
531 Recover an ONOS instance and verify the ONOS cluster can see the proper change
532 """
533 main.step( "Recovering ONOS instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700534 [ main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) for i in nodes ]
Jon Hall1efcb3f2016-08-23 13:42:15 -0700535 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700536 isUp = main.ONOSbench.isup( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700537 utilities.assert_equals( expect=main.TRUE, actual=isUp,
538 onpass="ONOS service is ready",
539 onfail="ONOS service did not start properly" )
540 for i in nodes:
541 main.step( "Checking if ONOS CLI is ready" )
Devin Lim142b5342017-07-20 15:22:39 -0700542 ctrl = main.Cluster.runningNodes[ i ]
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900543 # ctrl.CLI.startCellCli()
Devin Lim142b5342017-07-20 15:22:39 -0700544 cliResult = ctrl.CLI.startOnosCli( ctrl.ipAddress,
545 commandlineTimeout=60,
546 onosStartTimeout=100 )
547 ctrl.active = True
Jon Hall1efcb3f2016-08-23 13:42:15 -0700548 utilities.assert_equals( expect=main.TRUE,
549 actual=cliResult,
550 onpass="ONOS CLI is ready",
551 onfail="ONOS CLI is not ready" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700552
Pier3b58c652016-09-26 12:03:31 -0700553 main.step( "Checking ONOS nodes" )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700554 nodeResults = utilities.retry( main.Cluster.nodesCheck,
Pier3b58c652016-09-26 12:03:31 -0700555 False,
Pier3b58c652016-09-26 12:03:31 -0700556 attempts=5,
557 sleep=10 )
558 utilities.assert_equals( expect=True, actual=nodeResults,
559 onpass="Nodes check successful",
560 onfail="Nodes check NOT successful" )
561
562 if not nodeResults:
563 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700564 ctrl = main.Cluster.runningNodes[ i ]
Pier3b58c652016-09-26 12:03:31 -0700565 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700566 ctrl.name,
567 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Pier3b58c652016-09-26 12:03:31 -0700568 main.log.error( "Failed to start ONOS, stopping test" )
Devin Lim44075962017-08-11 10:56:37 -0700569 main.cleanAndExit()
Pier3b58c652016-09-26 12:03:31 -0700570
Devin Lim142b5342017-07-20 15:22:39 -0700571 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700572 main.FALSE,
573 kwargs={ 'numoswitch': switches,
574 'numolink': links,
575 'numoctrl': expNodes },
576 attempts=10,
577 sleep=12 )
578 utilities.assert_equals( expect=main.TRUE, actual=topology,
579 onpass="ONOS Instance down successful",
580 onfail="Failed to turn off ONOS Instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700581 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
582 main.FALSE,
583 attempts=10,
584 sleep=12 )
585 if ready:
586 ready = main.TRUE
587 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700588 onpass="ONOS summary command succeded",
589 onfail="ONOS summary command failed" )
590 if not ready:
591 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700592 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700593
594 @staticmethod
595 def addHostCfg( main ):
596 """
597 Adds Host Configuration to ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700598 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700599 """
600 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700601 hostCfg = {}
You Wangac02b142018-01-26 14:57:28 -0800602 with open( main.configPath + "/json/extra.json" ) as template:
Jon Hall1efcb3f2016-08-23 13:42:15 -0700603 hostCfg = json.load( template )
604 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'in1' ]
605 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700606 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700607 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
608 subjectClass="hosts",
609 subjectKey=urllib.quote( mac,
610 safe='' ),
611 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700612 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'out1' ]
613 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700614 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700615 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
616 subjectClass="hosts",
617 subjectKey=urllib.quote( mac,
618 safe='' ),
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700619 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700620 main.pingChart.update( { 'vlan1': { "expect": "True",
621 "hosts": [ "olt1", "vsg1" ] } } )
622 main.pingChart[ 'vlan5' ][ 'expect' ] = 0
623 main.pingChart[ 'vlan10' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700624 ports = "[%s,%s]" % ( 5, 6 )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700625 cfg = '{"of:0000000000000001":[{"vlan":1,"ports":%s,"name":"OLT 1"}]}' % ports
Devin Lim142b5342017-07-20 15:22:39 -0700626 main.Cluster.active( 0 ).REST.setNetCfg( json.loads( cfg ),
627 subjectClass="apps",
628 subjectKey="org.onosproject.segmentrouting",
629 configKey="xconnect" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700630
631 @staticmethod
632 def delHostCfg( main ):
633 """
634 Removest Host Configuration from ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700635 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700636 """
637 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700638 hostCfg = {}
You Wangac02b142018-01-26 14:57:28 -0800639 with open( main.configPath + "/json/extra.json" ) as template:
Jon Hall1efcb3f2016-08-23 13:42:15 -0700640 hostCfg = json.load( template )
641 main.step( "Removing host configuration" )
642 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700643 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700644 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
645 subjectKey=urllib.quote(
646 mac,
647 safe='' ),
648 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700649 main.step( "Removing configuration" )
650 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700651 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700652 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
653 subjectKey=urllib.quote(
654 mac,
655 safe='' ),
656 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700657 main.step( "Removing vlan configuration" )
658 main.pingChart[ 'vlan1' ][ 'expect' ] = 0
Devin Lim142b5342017-07-20 15:22:39 -0700659 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="apps",
660 subjectKey="org.onosproject.segmentrouting",
661 configKey="xconnect" )