blob: a5106be83317a1cd79c061dd98833bf9edbf8930 [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2016 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jon Hall1efcb3f2016-08-23 13:42:15 -070021import os
22import imp
23import time
24import json
25import urllib
26from core import utilities
27
28
29class Testcaselib:
Pierfb719b12016-09-19 14:51:44 -070030
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070031 useSSH = True
Pierfb719b12016-09-19 14:51:44 -070032
Jon Hall1efcb3f2016-08-23 13:42:15 -070033 @staticmethod
34 def initTest( main ):
35 """
36 - Construct tests variables
37 - GIT ( optional )
38 - Checkout ONOS master branch
39 - Pull latest ONOS code
40 - Building ONOS ( optional )
41 - Install ONOS package
42 - Build ONOS package
43 """
Devin Lim58046fa2017-07-05 16:55:00 -070044 try:
45 from tests.dependencies.ONOSSetup import ONOSSetup
46 main.testSetUp = ONOSSetup()
47 except ImportError:
48 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070049 main.cleanAndExit()
You Wangd5873482018-01-24 12:30:00 -080050 from tests.dependencies.Network import Network
51 main.Network = Network()
Devin Lim0c972b72018-02-08 14:53:59 -080052 main.testSetUp.envSetupDescription( False )
Devin Lim58046fa2017-07-05 16:55:00 -070053 stepResult = main.FALSE
54 try:
55 main.step( "Constructing test variables" )
56 # Test variables
57 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
58 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
Devin Lim58046fa2017-07-05 16:55:00 -070059 main.path = os.path.dirname( main.testFile )
You Wangac02b142018-01-26 14:57:28 -080060 main.topoPath = main.path + "/../dependencies/"
You Wang69362682018-02-12 13:35:54 -080061 main.configPath = main.path + "/../dependencies/conf/"
62 main.testConfPath = main.path + "/dependencies/conf/"
Devin Lim58046fa2017-07-05 16:55:00 -070063 main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
You Wangd87b2312018-01-30 12:47:17 -080064 main.topologyLib = main.params[ 'DEPENDENCY' ][ 'lib' ] if 'lib' in main.params[ 'DEPENDENCY' ] else None
65 main.topologyConf = main.params[ 'DEPENDENCY' ][ 'conf' ] if 'conf' in main.params[ 'DEPENDENCY' ] else None
You Wang69362682018-02-12 13:35:54 -080066 main.testConf = main.params[ 'DEPENDENCY' ][ 'testConf' ] if 'testConf' in main.params[ 'DEPENDENCY' ] else None
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070067 main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
Devin Lim58046fa2017-07-05 16:55:00 -070068 main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
Devin Lim58046fa2017-07-05 16:55:00 -070069 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
Jon Hall1efcb3f2016-08-23 13:42:15 -070070
Devin Lim0c972b72018-02-08 14:53:59 -080071 stepResult = main.testSetUp.envSetup( False )
Devin Lim58046fa2017-07-05 16:55:00 -070072 except Exception as e:
73 main.testSetUp.envSetupException( e )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -070074
Devin Lim58046fa2017-07-05 16:55:00 -070075 main.testSetUp.evnSetupConclusion( stepResult )
Jon Hall1efcb3f2016-08-23 13:42:15 -070076
Jon Hall1efcb3f2016-08-23 13:42:15 -070077 @staticmethod
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -080078 def installOnos( main, vlanCfg=True, skipPackage=False, cliSleep=10,
79 parallel=True ):
Jon Hall1efcb3f2016-08-23 13:42:15 -070080 """
81 - Set up cell
82 - Create cell file
83 - Set cell file
84 - Verify cell file
85 - Kill ONOS process
86 - Uninstall ONOS cluster
87 - Verify ONOS start up
88 - Install ONOS cluster
89 - Connect to cli
90 """
91 # main.scale[ 0 ] determines the current number of ONOS controller
You Wangd87b2312018-01-30 12:47:17 -080092 if not main.apps:
Jon Hall1efcb3f2016-08-23 13:42:15 -070093 main.log.error( "App list is empty" )
Devin Lim142b5342017-07-20 15:22:39 -070094 main.log.info( "NODE COUNT = " + str( main.Cluster.numCtrls ) )
95 main.log.info( ''.join( main.Cluster.getIps() ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -070096 main.dynamicHosts = [ 'in1', 'out1' ]
You Wanga0f6ff62018-01-11 15:46:30 -080097 main.testSetUp.ONOSSetUp( main.Cluster, newCell=True, cellName=main.cellName,
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -080098 skipPack=skipPackage,
99 useSSH=Testcaselib.useSSH,
Devin Lim0c972b72018-02-08 14:53:59 -0800100 installParallel=parallel, includeCaseDesc=False )
Devin Lim142b5342017-07-20 15:22:39 -0700101 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
102 main.FALSE,
You Wang1cdc5f52017-12-19 16:47:51 -0800103 sleep=cliSleep,
Devin Lim142b5342017-07-20 15:22:39 -0700104 attempts=10 )
105 if ready:
106 ready = main.TRUE
107 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700108 onpass="ONOS summary command succeded",
109 onfail="ONOS summary command failed" )
110
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -0800111 with open( "%s/json/%s.json" % (main.configPath, main.cfgName)) as cfg:
112 main.Cluster.active( 0 ).REST.setNetCfg(json.load(cfg))
113 with open("%s/json/%s.chart" % (main.configPath, main.cfgName)) as chart:
114 main.pingChart = json.load(chart)
Jon Hall1efcb3f2016-08-23 13:42:15 -0700115 if not ready:
116 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700117 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700118
Devin Lim142b5342017-07-20 15:22:39 -0700119 for ctrl in main.Cluster.active():
120 ctrl.CLI.logSet( "DEBUG", "org.onosproject.segmentrouting" )
121 ctrl.CLI.logSet( "DEBUG", "org.onosproject.driver.pipeline" )
122 ctrl.CLI.logSet( "DEBUG", "org.onosproject.store.group.impl" )
123 ctrl.CLI.logSet( "DEBUG", "org.onosproject.net.flowobjective.impl" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700124
125 @staticmethod
126 def startMininet( main, topology, args="" ):
You Wangd87b2312018-01-30 12:47:17 -0800127 copyResult = main.ONOSbench.scp( main.Mininet1,
128 main.topoPath + main.topology,
129 main.Mininet1.home,
130 direction="to" )
131 if main.topologyLib:
132 for lib in main.topologyLib.split(","):
133 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
134 main.topoPath + lib,
135 main.Mininet1.home,
136 direction="to" )
137 if main.topologyConf:
138 for conf in main.topologyConf.split(","):
139 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
You Wang69362682018-02-12 13:35:54 -0800140 main.configPath + conf,
141 "~/",
142 direction="to" )
143 if main.testConf:
144 for conf in main.testConf.split(","):
145 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
146 main.testConfPath + conf,
You Wangd87b2312018-01-30 12:47:17 -0800147 "~/",
148 direction="to" )
149 stepResult = copyResult
150 utilities.assert_equals( expect=main.TRUE,
151 actual=stepResult,
152 onpass="Successfully copied topo files",
153 onfail="Failed to copy topo files" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700154 main.step( "Starting Mininet Topology" )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700155 arg = "--onos-ip=%s %s" % (",".join([ctrl.ipAddress for ctrl in main.Cluster.runningNodes]), args)
Jon Hall1efcb3f2016-08-23 13:42:15 -0700156 main.topology = topology
157 topoResult = main.Mininet1.startNet(
158 topoFile=main.Mininet1.home + main.topology, args=arg )
159 stepResult = topoResult
160 utilities.assert_equals( expect=main.TRUE,
161 actual=stepResult,
162 onpass="Successfully loaded topology",
163 onfail="Failed to load topology" )
164 # Exit if topology did not load properly
165 if not topoResult:
Devin Lim44075962017-08-11 10:56:37 -0700166 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700167
168 @staticmethod
Devin Lim142b5342017-07-20 15:22:39 -0700169 def config( main, cfgName ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700170 main.spines = []
Piera2a7e1b2016-10-04 11:51:43 -0700171
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700172 main.failures = int( main.params[ 'failures' ] )
173 main.cfgName = cfgName
Piera2a7e1b2016-10-04 11:51:43 -0700174
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700175 if main.cfgName == '2x2':
176 spine = {}
177 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
178 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
179 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700180
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700181 spine = {}
182 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
183 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
184 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700185
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700186 elif main.cfgName == '4x4':
187 spine = {}
188 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
189 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
190 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700191
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700192 spine = {}
193 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
194 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
195 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700196
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700197 spine = {}
198 spine[ 'name' ] = main.params[ 'switches' ][ 'spine3' ]
199 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid3' ]
200 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700201
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700202 spine = {}
203 spine[ 'name' ] = main.params[ 'switches' ][ 'spine4' ]
204 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid4' ]
205 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700206
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700207 else:
Piera2a7e1b2016-10-04 11:51:43 -0700208 main.log.error( "Configuration failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700209 main.cleanAndExit()
Piera2a7e1b2016-10-04 11:51:43 -0700210
211 @staticmethod
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900212 def checkFlows( main, minFlowCount, tag="", dumpflows=True, sleep=10 ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700213 main.step(
214 " Check whether the flow count is bigger than %s" % minFlowCount )
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900215 if tag == "":
216 tag = 'CASE%d' % main.CurrentTestCaseNumber
Devin Lim142b5342017-07-20 15:22:39 -0700217 count = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowCount,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700218 main.FALSE,
219 kwargs={ 'min': minFlowCount },
220 attempts=10,
You Wang1cdc5f52017-12-19 16:47:51 -0800221 sleep=sleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700222 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700223 expect=True,
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700224 actual=( count > 0 ),
Jon Hall1efcb3f2016-08-23 13:42:15 -0700225 onpass="Flow count looks correct: " + str( count ),
226 onfail="Flow count looks wrong: " + str( count ) )
227
228 main.step( "Check whether all flow status are ADDED" )
Devin Lim142b5342017-07-20 15:22:39 -0700229 flowCheck = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowsState,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700230 main.FALSE,
231 kwargs={ 'isPENDING': False },
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800232 attempts=5,
You Wang1cdc5f52017-12-19 16:47:51 -0800233 sleep=sleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700234 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700235 expect=main.TRUE,
236 actual=flowCheck,
237 onpass="Flow status is correct!",
238 onfail="Flow status is wrong!" )
239 if dumpflows:
Devin Lim142b5342017-07-20 15:22:39 -0700240 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700241 "flows",
242 main.logdir,
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900243 tag + "_FlowsBefore" )
Devin Lim142b5342017-07-20 15:22:39 -0700244 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700245 "groups",
246 main.logdir,
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900247 tag + "_GroupsBefore" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700248
249 @staticmethod
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800250 def checkFlowsByDpid( main, dpid, minFlowCount, sleep=10 ):
251 main.step(
252 " Check whether the flow count of device %s is bigger than %s" % ( dpid, minFlowCount ) )
253 count = utilities.retry( main.Cluster.active( 0 ).CLI.flowAddedCount,
254 None,
255 args=( dpid, ),
256 attempts=5,
257 sleep=sleep )
258 utilities.assertEquals(
259 expect=True,
260 actual=( int( count ) > minFlowCount ),
261 onpass="Flow count looks correct: " + count ,
262 onfail="Flow count looks wrong: " + count )
263
264 @staticmethod
Andreas Pantelopoulosf6ed5012018-02-08 21:26:01 -0800265 def pingAllBasedOnIp( main, tag="", dumpflows=True ):
266 main.log.report( "Check full connectivity" )
267 print main.pingChart
268 if tag == "":
269 tag = 'CASE%d' % main.CurrentTestCaseNumber
270 for entry in main.pingChart.itervalues():
271 print entry
272 hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
273 try:
274 expect = main.TRUE if str(expect).lower() == 'true' else main.FALSE
275 except:
276 expect = main.FALSE
277 main.step( "Connectivity for %s %s" % ( str( hosts ), tag ) )
278
279 if ("v4" in hosts[0]):
280 pa = main.Network.pingallHosts( hosts )
281 utilities.assert_equals( expect=expect, actual=pa,
282 onpass="IPv4 connectivity successfully tested",
283 onfail="IPv4 connectivity failed" )
284 if ("v6" in hosts[0]):
285 pa = main.Network.pingIpv6Hosts( hosts )
286 utilities.assert_equals( expect=expect, actual=pa,
287 onpass="IPv6 connectivity successfully tested",
288 onfail="IPv6 connectivity failed" )
289
290 if dumpflows:
291 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
292 "flows",
293 main.logdir,
294 tag + "_FlowsOn" )
295 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
296 "groups",
297 main.logdir,
298 tag + "_GroupsOn" )
299
300 @staticmethod
Jon Hall1efcb3f2016-08-23 13:42:15 -0700301 def pingAll( main, tag="", dumpflows=True ):
302 main.log.report( "Check full connectivity" )
303 print main.pingChart
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900304 if tag == "":
305 tag = 'CASE%d' % main.CurrentTestCaseNumber
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700306 for entry in main.pingChart.itervalues():
Jon Hall1efcb3f2016-08-23 13:42:15 -0700307 print entry
308 hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700309 try:
310 expect = main.TRUE if str(expect).lower() == 'true' else main.FALSE
311 except:
312 expect = main.FALSE
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700313 main.step( "Connectivity for %s %s" % ( str( hosts ), tag ) )
You Wangd5873482018-01-24 12:30:00 -0800314 pa = main.Network.pingallHosts( hosts )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700315
Jon Hall1efcb3f2016-08-23 13:42:15 -0700316 utilities.assert_equals( expect=expect, actual=pa,
317 onpass="IP connectivity successfully tested",
318 onfail="IP connectivity failed" )
319 if dumpflows:
Devin Lim142b5342017-07-20 15:22:39 -0700320 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700321 "flows",
322 main.logdir,
Devin Lim97b6b862018-01-23 22:51:25 -0800323 tag + "_FlowsOn" )
Devin Lim142b5342017-07-20 15:22:39 -0700324 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700325 "groups",
326 main.logdir,
Devin Lim97b6b862018-01-23 22:51:25 -0800327 tag + "_GroupsOn" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700328
329 @staticmethod
330 def killLink( main, end1, end2, switches, links ):
331 """
332 end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
333 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
334 Kill a link and verify ONOS can see the proper link change
335 """
336 main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700337 main.step( "Kill link between %s and %s" % ( end1, end2 ) )
You Wangd5873482018-01-24 12:30:00 -0800338 LinkDown = main.Network.link( END1=end1, END2=end2, OPTION="down" )
339 LinkDown = main.Network.link( END2=end1, END1=end2, OPTION="down" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700340 main.log.info(
341 "Waiting %s seconds for link down to be discovered" % main.linkSleep )
342 time.sleep( main.linkSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700343 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700344 main.FALSE,
345 kwargs={ 'numoswitch': switches,
346 'numolink': links },
347 attempts=10,
348 sleep=main.linkSleep )
349 result = topology & LinkDown
350 utilities.assert_equals( expect=main.TRUE, actual=result,
351 onpass="Link down successful",
352 onfail="Failed to turn off link?" )
353
354 @staticmethod
355 def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
356 links ):
357 """
358 Params:
359 end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
360 dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
361 port1, port2: respective port of the end switches that connects to the link, ex.:'1'
362 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
363 Kill a link and verify ONOS can see the proper link change
364 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700365 main.step( "Restore link between %s and %s" % ( end1, end2 ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700366 result = False
367 count = 0
368 while True:
369 count += 1
You Wangd5873482018-01-24 12:30:00 -0800370 main.Network.link( END1=end1, END2=end2, OPTION="up" )
371 main.Network.link( END2=end1, END1=end2, OPTION="up" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700372 main.log.info(
373 "Waiting %s seconds for link up to be discovered" % main.linkSleep )
374 time.sleep( main.linkSleep )
Pierfb719b12016-09-19 14:51:44 -0700375
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700376 for i in range( 0, main.Cluster.numCtrls ):
Devin Lim142b5342017-07-20 15:22:39 -0700377 ctrl = main.Cluster.runningNodes[ i ]
378 onosIsUp = main.ONOSbench.isup( ctrl.ipAddress )
Pierfb719b12016-09-19 14:51:44 -0700379 if onosIsUp == main.TRUE:
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900380 ctrl.CLI.portstate( dpid=dpid1, port=port1, state='Enable' )
381 ctrl.CLI.portstate( dpid=dpid2, port=port2, state='Enable' )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700382 time.sleep( main.linkSleep )
383
Devin Lim142b5342017-07-20 15:22:39 -0700384 result = main.Cluster.active( 0 ).CLI.checkStatus( numoswitch=switches,
385 numolink=links )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700386 if count > 5 or result:
387 break
388 utilities.assert_equals( expect=main.TRUE, actual=result,
389 onpass="Link up successful",
390 onfail="Failed to bring link up" )
391
392 @staticmethod
393 def killSwitch( main, switch, switches, links ):
394 """
395 Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
396 Completely kill a switch and verify ONOS can see the proper change
397 """
398 main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
399 main.step( "Kill " + switch )
400 main.log.info( "Stopping" + switch )
You Wangd5873482018-01-24 12:30:00 -0800401 main.Network.switch( SW=switch, OPTION="stop" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700402 # todo make this repeatable
403 main.log.info( "Waiting %s seconds for switch down to be discovered" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700404 main.switchSleep ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700405 time.sleep( main.switchSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700406 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700407 main.FALSE,
408 kwargs={ 'numoswitch': switches,
409 'numolink': links },
410 attempts=10,
411 sleep=main.switchSleep )
412 utilities.assert_equals( expect=main.TRUE, actual=topology,
413 onpass="Kill switch successful",
414 onfail="Failed to kill switch?" )
415
416 @staticmethod
417 def recoverSwitch( main, switch, switches, links ):
418 """
419 Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
420 Recover a switch and verify ONOS can see the proper change
421 """
422 # todo make this repeatable
423 main.step( "Recovering " + switch )
424 main.log.info( "Starting" + switch )
You Wangd5873482018-01-24 12:30:00 -0800425 main.Network.switch( SW=switch, OPTION="start" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700426 main.log.info( "Waiting %s seconds for switch up to be discovered" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700427 main.switchSleep ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700428 time.sleep( main.switchSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700429 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700430 main.FALSE,
431 kwargs={ 'numoswitch': switches,
432 'numolink': links },
433 attempts=10,
434 sleep=main.switchSleep )
435 utilities.assert_equals( expect=main.TRUE, actual=topology,
436 onpass="Switch recovery successful",
437 onfail="Failed to recover switch?" )
438
439 @staticmethod
440 def cleanup( main ):
441 """
442 Stop Onos-cluster.
443 Stops Mininet
444 Copies ONOS log
445 """
Devin Lim58046fa2017-07-05 16:55:00 -0700446 try:
447 from tests.dependencies.utils import Utils
448 except ImportError:
449 main.log.error( "Utils not found exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -0700450 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -0700451 try:
Devin Lim142b5342017-07-20 15:22:39 -0700452 main.utils
Devin Lim58046fa2017-07-05 16:55:00 -0700453 except ( NameError, AttributeError ):
Devin Lim142b5342017-07-20 15:22:39 -0700454 main.utils = Utils()
Devin Lim58046fa2017-07-05 16:55:00 -0700455
456 main.utils.mininetCleanup( main.Mininet1 )
457
Devin Lim0c972b72018-02-08 14:53:59 -0800458 main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True, includeCaseDesc=False )
Devin Lim58046fa2017-07-05 16:55:00 -0700459
Devin Lim142b5342017-07-20 15:22:39 -0700460 for ctrl in main.Cluster.active():
461 main.ONOSbench.onosStop( ctrl.ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700462
463 @staticmethod
464 def killOnos( main, nodes, switches, links, expNodes ):
465 """
466 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
467 switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
468 Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
469 """
470 main.step( "Killing ONOS instance" )
Pier3b58c652016-09-26 12:03:31 -0700471
Jon Hall1efcb3f2016-08-23 13:42:15 -0700472 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700473 killResult = main.ONOSbench.onosDie( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700474 utilities.assert_equals( expect=main.TRUE, actual=killResult,
475 onpass="ONOS instance Killed",
476 onfail="Error killing ONOS instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700477 main.Cluster.runningNodes[ i ].active = False
Jon Hall1efcb3f2016-08-23 13:42:15 -0700478 time.sleep( 12 )
Pier3b58c652016-09-26 12:03:31 -0700479
Devin Lim142b5342017-07-20 15:22:39 -0700480 if len( nodes ) < main.Cluster.numCtrls:
Pier3b58c652016-09-26 12:03:31 -0700481
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700482 nodeResults = utilities.retry( main.Cluster.nodesCheck,
Pier3b58c652016-09-26 12:03:31 -0700483 False,
Pier3b58c652016-09-26 12:03:31 -0700484 attempts=5,
485 sleep=10 )
486 utilities.assert_equals( expect=True, actual=nodeResults,
487 onpass="Nodes check successful",
488 onfail="Nodes check NOT successful" )
489
490 if not nodeResults:
491 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700492 ctrl = main.Cluster.runningNodes[ i ]
Pier3b58c652016-09-26 12:03:31 -0700493 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700494 ctrl.name,
495 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Pier3b58c652016-09-26 12:03:31 -0700496 main.log.error( "Failed to kill ONOS, stopping test" )
Devin Lim44075962017-08-11 10:56:37 -0700497 main.cleanAndExit()
Pier3b58c652016-09-26 12:03:31 -0700498
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900499 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700500 main.FALSE,
501 kwargs={ 'numoswitch': switches,
502 'numolink': links,
503 'numoctrl': expNodes },
504 attempts=10,
505 sleep=12 )
506 utilities.assert_equals( expect=main.TRUE, actual=topology,
507 onpass="ONOS Instance down successful",
508 onfail="Failed to turn off ONOS Instance" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700509
510 @staticmethod
511 def recoverOnos( main, nodes, switches, links, expNodes ):
512 """
513 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
514 switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
515 Recover an ONOS instance and verify the ONOS cluster can see the proper change
516 """
517 main.step( "Recovering ONOS instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700518 [ main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) for i in nodes ]
Jon Hall1efcb3f2016-08-23 13:42:15 -0700519 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700520 isUp = main.ONOSbench.isup( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700521 utilities.assert_equals( expect=main.TRUE, actual=isUp,
522 onpass="ONOS service is ready",
523 onfail="ONOS service did not start properly" )
524 for i in nodes:
525 main.step( "Checking if ONOS CLI is ready" )
Devin Lim142b5342017-07-20 15:22:39 -0700526 ctrl = main.Cluster.runningNodes[ i ]
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900527 # ctrl.CLI.startCellCli()
Devin Lim142b5342017-07-20 15:22:39 -0700528 cliResult = ctrl.CLI.startOnosCli( ctrl.ipAddress,
529 commandlineTimeout=60,
530 onosStartTimeout=100 )
531 ctrl.active = True
Jon Hall1efcb3f2016-08-23 13:42:15 -0700532 utilities.assert_equals( expect=main.TRUE,
533 actual=cliResult,
534 onpass="ONOS CLI is ready",
535 onfail="ONOS CLI is not ready" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700536
Pier3b58c652016-09-26 12:03:31 -0700537 main.step( "Checking ONOS nodes" )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700538 nodeResults = utilities.retry( main.Cluster.nodesCheck,
Pier3b58c652016-09-26 12:03:31 -0700539 False,
Pier3b58c652016-09-26 12:03:31 -0700540 attempts=5,
541 sleep=10 )
542 utilities.assert_equals( expect=True, actual=nodeResults,
543 onpass="Nodes check successful",
544 onfail="Nodes check NOT successful" )
545
546 if not nodeResults:
547 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700548 ctrl = main.Cluster.runningNodes[ i ]
Pier3b58c652016-09-26 12:03:31 -0700549 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700550 ctrl.name,
551 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Pier3b58c652016-09-26 12:03:31 -0700552 main.log.error( "Failed to start ONOS, stopping test" )
Devin Lim44075962017-08-11 10:56:37 -0700553 main.cleanAndExit()
Pier3b58c652016-09-26 12:03:31 -0700554
Devin Lim142b5342017-07-20 15:22:39 -0700555 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700556 main.FALSE,
557 kwargs={ 'numoswitch': switches,
558 'numolink': links,
559 'numoctrl': expNodes },
560 attempts=10,
561 sleep=12 )
562 utilities.assert_equals( expect=main.TRUE, actual=topology,
563 onpass="ONOS Instance down successful",
564 onfail="Failed to turn off ONOS Instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700565 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
566 main.FALSE,
567 attempts=10,
568 sleep=12 )
569 if ready:
570 ready = main.TRUE
571 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700572 onpass="ONOS summary command succeded",
573 onfail="ONOS summary command failed" )
574 if not ready:
575 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700576 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700577
578 @staticmethod
579 def addHostCfg( main ):
580 """
581 Adds Host Configuration to ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700582 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700583 """
584 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700585 hostCfg = {}
You Wangac02b142018-01-26 14:57:28 -0800586 with open( main.configPath + "/json/extra.json" ) as template:
Jon Hall1efcb3f2016-08-23 13:42:15 -0700587 hostCfg = json.load( template )
588 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'in1' ]
589 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700590 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700591 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
592 subjectClass="hosts",
593 subjectKey=urllib.quote( mac,
594 safe='' ),
595 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700596 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'out1' ]
597 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700598 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700599 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
600 subjectClass="hosts",
601 subjectKey=urllib.quote( mac,
602 safe='' ),
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700603 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700604 main.pingChart.update( { 'vlan1': { "expect": "True",
605 "hosts": [ "olt1", "vsg1" ] } } )
606 main.pingChart[ 'vlan5' ][ 'expect' ] = 0
607 main.pingChart[ 'vlan10' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700608 ports = "[%s,%s]" % ( 5, 6 )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700609 cfg = '{"of:0000000000000001":[{"vlan":1,"ports":%s,"name":"OLT 1"}]}' % ports
Devin Lim142b5342017-07-20 15:22:39 -0700610 main.Cluster.active( 0 ).REST.setNetCfg( json.loads( cfg ),
611 subjectClass="apps",
612 subjectKey="org.onosproject.segmentrouting",
613 configKey="xconnect" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700614
615 @staticmethod
616 def delHostCfg( main ):
617 """
618 Removest Host Configuration from ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700619 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700620 """
621 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700622 hostCfg = {}
You Wangac02b142018-01-26 14:57:28 -0800623 with open( main.configPath + "/json/extra.json" ) as template:
Jon Hall1efcb3f2016-08-23 13:42:15 -0700624 hostCfg = json.load( template )
625 main.step( "Removing host configuration" )
626 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700627 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700628 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
629 subjectKey=urllib.quote(
630 mac,
631 safe='' ),
632 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700633 main.step( "Removing configuration" )
634 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700635 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700636 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
637 subjectKey=urllib.quote(
638 mac,
639 safe='' ),
640 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700641 main.step( "Removing vlan configuration" )
642 main.pingChart[ 'vlan1' ][ 'expect' ] = 0
Devin Lim142b5342017-07-20 15:22:39 -0700643 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="apps",
644 subjectKey="org.onosproject.segmentrouting",
645 configKey="xconnect" )