blob: 22d21638cd52c60594333f98c58bc7ac86408910 [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2016 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jon Hall1efcb3f2016-08-23 13:42:15 -070021import os
22import imp
23import time
24import json
25import urllib
26from core import utilities
27
28
29class Testcaselib:
Pierfb719b12016-09-19 14:51:44 -070030
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070031 useSSH = True
Pierfb719b12016-09-19 14:51:44 -070032
Jon Hall1efcb3f2016-08-23 13:42:15 -070033 @staticmethod
34 def initTest( main ):
35 """
36 - Construct tests variables
37 - GIT ( optional )
38 - Checkout ONOS master branch
39 - Pull latest ONOS code
40 - Building ONOS ( optional )
41 - Install ONOS package
42 - Build ONOS package
43 """
Devin Lim58046fa2017-07-05 16:55:00 -070044 try:
45 from tests.dependencies.ONOSSetup import ONOSSetup
46 main.testSetUp = ONOSSetup()
47 except ImportError:
48 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070049 main.cleanAndExit()
You Wangd5873482018-01-24 12:30:00 -080050 from tests.dependencies.Network import Network
51 main.Network = Network()
Devin Lim58046fa2017-07-05 16:55:00 -070052 main.testSetUp.envSetupDescription()
53 stepResult = main.FALSE
54 try:
55 main.step( "Constructing test variables" )
56 # Test variables
57 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
58 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
Devin Lim58046fa2017-07-05 16:55:00 -070059 main.path = os.path.dirname( main.testFile )
You Wangac02b142018-01-26 14:57:28 -080060 main.topoPath = main.path + "/../dependencies/"
61 main.configPath = main.path + "/../dependencies/"
Devin Lim58046fa2017-07-05 16:55:00 -070062 main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
You Wangd87b2312018-01-30 12:47:17 -080063 main.topologyLib = main.params[ 'DEPENDENCY' ][ 'lib' ] if 'lib' in main.params[ 'DEPENDENCY' ] else None
64 main.topologyConf = main.params[ 'DEPENDENCY' ][ 'conf' ] if 'conf' in main.params[ 'DEPENDENCY' ] else None
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070065 main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
Devin Lim58046fa2017-07-05 16:55:00 -070066 main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
Devin Lim58046fa2017-07-05 16:55:00 -070067 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
Jon Hall1efcb3f2016-08-23 13:42:15 -070068
Devin Lim142b5342017-07-20 15:22:39 -070069 stepResult = main.testSetUp.envSetup()
Devin Lim58046fa2017-07-05 16:55:00 -070070 except Exception as e:
71 main.testSetUp.envSetupException( e )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -070072
Devin Lim58046fa2017-07-05 16:55:00 -070073 main.testSetUp.evnSetupConclusion( stepResult )
Jon Hall1efcb3f2016-08-23 13:42:15 -070074
Jon Hall1efcb3f2016-08-23 13:42:15 -070075 @staticmethod
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -080076 def installOnos( main, vlanCfg=True, skipPackage=False, cliSleep=10,
77 parallel=True ):
Jon Hall1efcb3f2016-08-23 13:42:15 -070078 """
79 - Set up cell
80 - Create cell file
81 - Set cell file
82 - Verify cell file
83 - Kill ONOS process
84 - Uninstall ONOS cluster
85 - Verify ONOS start up
86 - Install ONOS cluster
87 - Connect to cli
88 """
89 # main.scale[ 0 ] determines the current number of ONOS controller
You Wangd87b2312018-01-30 12:47:17 -080090 if not main.apps:
Jon Hall1efcb3f2016-08-23 13:42:15 -070091 main.log.error( "App list is empty" )
Devin Lim142b5342017-07-20 15:22:39 -070092 main.log.info( "NODE COUNT = " + str( main.Cluster.numCtrls ) )
93 main.log.info( ''.join( main.Cluster.getIps() ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -070094 main.dynamicHosts = [ 'in1', 'out1' ]
You Wanga0f6ff62018-01-11 15:46:30 -080095 main.testSetUp.ONOSSetUp( main.Cluster, newCell=True, cellName=main.cellName,
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -080096 skipPack=skipPackage,
97 useSSH=Testcaselib.useSSH,
98 installParallel=parallel)
Devin Lim142b5342017-07-20 15:22:39 -070099 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
100 main.FALSE,
You Wang1cdc5f52017-12-19 16:47:51 -0800101 sleep=cliSleep,
Devin Lim142b5342017-07-20 15:22:39 -0700102 attempts=10 )
103 if ready:
104 ready = main.TRUE
105 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700106 onpass="ONOS summary command succeded",
107 onfail="ONOS summary command failed" )
108
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -0800109 with open( "%s/json/%s.json" % (main.configPath, main.cfgName)) as cfg:
110 main.Cluster.active( 0 ).REST.setNetCfg(json.load(cfg))
111 with open("%s/json/%s.chart" % (main.configPath, main.cfgName)) as chart:
112 main.pingChart = json.load(chart)
Jon Hall1efcb3f2016-08-23 13:42:15 -0700113 if not ready:
114 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700115 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700116
Devin Lim142b5342017-07-20 15:22:39 -0700117 for ctrl in main.Cluster.active():
118 ctrl.CLI.logSet( "DEBUG", "org.onosproject.segmentrouting" )
119 ctrl.CLI.logSet( "DEBUG", "org.onosproject.driver.pipeline" )
120 ctrl.CLI.logSet( "DEBUG", "org.onosproject.store.group.impl" )
121 ctrl.CLI.logSet( "DEBUG", "org.onosproject.net.flowobjective.impl" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700122
123 @staticmethod
124 def startMininet( main, topology, args="" ):
You Wangd87b2312018-01-30 12:47:17 -0800125 copyResult = main.ONOSbench.scp( main.Mininet1,
126 main.topoPath + main.topology,
127 main.Mininet1.home,
128 direction="to" )
129 if main.topologyLib:
130 for lib in main.topologyLib.split(","):
131 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
132 main.topoPath + lib,
133 main.Mininet1.home,
134 direction="to" )
135 if main.topologyConf:
136 for conf in main.topologyConf.split(","):
137 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
138 main.topoPath + "conf/" + conf,
139 "~/",
140 direction="to" )
141 stepResult = copyResult
142 utilities.assert_equals( expect=main.TRUE,
143 actual=stepResult,
144 onpass="Successfully copied topo files",
145 onfail="Failed to copy topo files" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700146 main.step( "Starting Mininet Topology" )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700147 arg = "--onos-ip=%s %s" % (",".join([ctrl.ipAddress for ctrl in main.Cluster.runningNodes]), args)
Jon Hall1efcb3f2016-08-23 13:42:15 -0700148 main.topology = topology
149 topoResult = main.Mininet1.startNet(
150 topoFile=main.Mininet1.home + main.topology, args=arg )
151 stepResult = topoResult
152 utilities.assert_equals( expect=main.TRUE,
153 actual=stepResult,
154 onpass="Successfully loaded topology",
155 onfail="Failed to load topology" )
156 # Exit if topology did not load properly
157 if not topoResult:
Devin Lim44075962017-08-11 10:56:37 -0700158 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700159
160 @staticmethod
Devin Lim142b5342017-07-20 15:22:39 -0700161 def config( main, cfgName ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700162 main.spines = []
Piera2a7e1b2016-10-04 11:51:43 -0700163
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700164 main.failures = int( main.params[ 'failures' ] )
165 main.cfgName = cfgName
Piera2a7e1b2016-10-04 11:51:43 -0700166
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700167 if main.cfgName == '2x2':
168 spine = {}
169 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
170 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
171 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700172
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700173 spine = {}
174 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
175 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
176 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700177
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700178 elif main.cfgName == '4x4':
179 spine = {}
180 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
181 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
182 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700183
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700184 spine = {}
185 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
186 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
187 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700188
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700189 spine = {}
190 spine[ 'name' ] = main.params[ 'switches' ][ 'spine3' ]
191 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid3' ]
192 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700193
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700194 spine = {}
195 spine[ 'name' ] = main.params[ 'switches' ][ 'spine4' ]
196 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid4' ]
197 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700198
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700199 else:
Piera2a7e1b2016-10-04 11:51:43 -0700200 main.log.error( "Configuration failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700201 main.cleanAndExit()
Piera2a7e1b2016-10-04 11:51:43 -0700202
203 @staticmethod
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900204 def checkFlows( main, minFlowCount, tag="", dumpflows=True, sleep=10 ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700205 main.step(
206 " Check whether the flow count is bigger than %s" % minFlowCount )
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900207 if tag == "":
208 tag = 'CASE%d' % main.CurrentTestCaseNumber
Devin Lim142b5342017-07-20 15:22:39 -0700209 count = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowCount,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700210 main.FALSE,
211 kwargs={ 'min': minFlowCount },
212 attempts=10,
You Wang1cdc5f52017-12-19 16:47:51 -0800213 sleep=sleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700214 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700215 expect=True,
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700216 actual=( count > 0 ),
Jon Hall1efcb3f2016-08-23 13:42:15 -0700217 onpass="Flow count looks correct: " + str( count ),
218 onfail="Flow count looks wrong: " + str( count ) )
219
220 main.step( "Check whether all flow status are ADDED" )
Devin Lim142b5342017-07-20 15:22:39 -0700221 flowCheck = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowsState,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700222 main.FALSE,
223 kwargs={ 'isPENDING': False },
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800224 attempts=5,
You Wang1cdc5f52017-12-19 16:47:51 -0800225 sleep=sleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700226 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700227 expect=main.TRUE,
228 actual=flowCheck,
229 onpass="Flow status is correct!",
230 onfail="Flow status is wrong!" )
231 if dumpflows:
Devin Lim142b5342017-07-20 15:22:39 -0700232 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700233 "flows",
234 main.logdir,
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900235 tag + "_FlowsBefore" )
Devin Lim142b5342017-07-20 15:22:39 -0700236 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700237 "groups",
238 main.logdir,
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900239 tag + "_GroupsBefore" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700240
241 @staticmethod
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800242 def checkFlowsByDpid( main, dpid, minFlowCount, sleep=10 ):
243 main.step(
244 " Check whether the flow count of device %s is bigger than %s" % ( dpid, minFlowCount ) )
245 count = utilities.retry( main.Cluster.active( 0 ).CLI.flowAddedCount,
246 None,
247 args=( dpid, ),
248 attempts=5,
249 sleep=sleep )
250 utilities.assertEquals(
251 expect=True,
252 actual=( int( count ) > minFlowCount ),
253 onpass="Flow count looks correct: " + count ,
254 onfail="Flow count looks wrong: " + count )
255
256 @staticmethod
Andreas Pantelopoulosf6ed5012018-02-08 21:26:01 -0800257 def pingAllBasedOnIp( main, tag="", dumpflows=True ):
258 main.log.report( "Check full connectivity" )
259 print main.pingChart
260 if tag == "":
261 tag = 'CASE%d' % main.CurrentTestCaseNumber
262 for entry in main.pingChart.itervalues():
263 print entry
264 hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
265 try:
266 expect = main.TRUE if str(expect).lower() == 'true' else main.FALSE
267 except:
268 expect = main.FALSE
269 main.step( "Connectivity for %s %s" % ( str( hosts ), tag ) )
270
271 if ("v4" in hosts[0]):
272 pa = main.Network.pingallHosts( hosts )
273 utilities.assert_equals( expect=expect, actual=pa,
274 onpass="IPv4 connectivity successfully tested",
275 onfail="IPv4 connectivity failed" )
276 if ("v6" in hosts[0]):
277 pa = main.Network.pingIpv6Hosts( hosts )
278 utilities.assert_equals( expect=expect, actual=pa,
279 onpass="IPv6 connectivity successfully tested",
280 onfail="IPv6 connectivity failed" )
281
282 if dumpflows:
283 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
284 "flows",
285 main.logdir,
286 tag + "_FlowsOn" )
287 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
288 "groups",
289 main.logdir,
290 tag + "_GroupsOn" )
291
292 @staticmethod
Jon Hall1efcb3f2016-08-23 13:42:15 -0700293 def pingAll( main, tag="", dumpflows=True ):
294 main.log.report( "Check full connectivity" )
295 print main.pingChart
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900296 if tag == "":
297 tag = 'CASE%d' % main.CurrentTestCaseNumber
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700298 for entry in main.pingChart.itervalues():
Jon Hall1efcb3f2016-08-23 13:42:15 -0700299 print entry
300 hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700301 try:
302 expect = main.TRUE if str(expect).lower() == 'true' else main.FALSE
303 except:
304 expect = main.FALSE
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700305 main.step( "Connectivity for %s %s" % ( str( hosts ), tag ) )
You Wangd5873482018-01-24 12:30:00 -0800306 pa = main.Network.pingallHosts( hosts )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700307
Jon Hall1efcb3f2016-08-23 13:42:15 -0700308 utilities.assert_equals( expect=expect, actual=pa,
309 onpass="IP connectivity successfully tested",
310 onfail="IP connectivity failed" )
311 if dumpflows:
Devin Lim142b5342017-07-20 15:22:39 -0700312 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700313 "flows",
314 main.logdir,
Devin Lim97b6b862018-01-23 22:51:25 -0800315 tag + "_FlowsOn" )
Devin Lim142b5342017-07-20 15:22:39 -0700316 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700317 "groups",
318 main.logdir,
Devin Lim97b6b862018-01-23 22:51:25 -0800319 tag + "_GroupsOn" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700320
321 @staticmethod
322 def killLink( main, end1, end2, switches, links ):
323 """
324 end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
325 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
326 Kill a link and verify ONOS can see the proper link change
327 """
328 main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700329 main.step( "Kill link between %s and %s" % ( end1, end2 ) )
You Wangd5873482018-01-24 12:30:00 -0800330 LinkDown = main.Network.link( END1=end1, END2=end2, OPTION="down" )
331 LinkDown = main.Network.link( END2=end1, END1=end2, OPTION="down" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700332 main.log.info(
333 "Waiting %s seconds for link down to be discovered" % main.linkSleep )
334 time.sleep( main.linkSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700335 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700336 main.FALSE,
337 kwargs={ 'numoswitch': switches,
338 'numolink': links },
339 attempts=10,
340 sleep=main.linkSleep )
341 result = topology & LinkDown
342 utilities.assert_equals( expect=main.TRUE, actual=result,
343 onpass="Link down successful",
344 onfail="Failed to turn off link?" )
345
346 @staticmethod
347 def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
348 links ):
349 """
350 Params:
351 end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
352 dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
353 port1, port2: respective port of the end switches that connects to the link, ex.:'1'
354 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
355 Kill a link and verify ONOS can see the proper link change
356 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700357 main.step( "Restore link between %s and %s" % ( end1, end2 ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700358 result = False
359 count = 0
360 while True:
361 count += 1
You Wangd5873482018-01-24 12:30:00 -0800362 main.Network.link( END1=end1, END2=end2, OPTION="up" )
363 main.Network.link( END2=end1, END1=end2, OPTION="up" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700364 main.log.info(
365 "Waiting %s seconds for link up to be discovered" % main.linkSleep )
366 time.sleep( main.linkSleep )
Pierfb719b12016-09-19 14:51:44 -0700367
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700368 for i in range( 0, main.Cluster.numCtrls ):
Devin Lim142b5342017-07-20 15:22:39 -0700369 ctrl = main.Cluster.runningNodes[ i ]
370 onosIsUp = main.ONOSbench.isup( ctrl.ipAddress )
Pierfb719b12016-09-19 14:51:44 -0700371 if onosIsUp == main.TRUE:
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900372 ctrl.CLI.portstate( dpid=dpid1, port=port1, state='Enable' )
373 ctrl.CLI.portstate( dpid=dpid2, port=port2, state='Enable' )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700374 time.sleep( main.linkSleep )
375
Devin Lim142b5342017-07-20 15:22:39 -0700376 result = main.Cluster.active( 0 ).CLI.checkStatus( numoswitch=switches,
377 numolink=links )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700378 if count > 5 or result:
379 break
380 utilities.assert_equals( expect=main.TRUE, actual=result,
381 onpass="Link up successful",
382 onfail="Failed to bring link up" )
383
384 @staticmethod
385 def killSwitch( main, switch, switches, links ):
386 """
387 Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
388 Completely kill a switch and verify ONOS can see the proper change
389 """
390 main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
391 main.step( "Kill " + switch )
392 main.log.info( "Stopping" + switch )
You Wangd5873482018-01-24 12:30:00 -0800393 main.Network.switch( SW=switch, OPTION="stop" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700394 # todo make this repeatable
395 main.log.info( "Waiting %s seconds for switch down to be discovered" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700396 main.switchSleep ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700397 time.sleep( main.switchSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700398 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700399 main.FALSE,
400 kwargs={ 'numoswitch': switches,
401 'numolink': links },
402 attempts=10,
403 sleep=main.switchSleep )
404 utilities.assert_equals( expect=main.TRUE, actual=topology,
405 onpass="Kill switch successful",
406 onfail="Failed to kill switch?" )
407
408 @staticmethod
409 def recoverSwitch( main, switch, switches, links ):
410 """
411 Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
412 Recover a switch and verify ONOS can see the proper change
413 """
414 # todo make this repeatable
415 main.step( "Recovering " + switch )
416 main.log.info( "Starting" + switch )
You Wangd5873482018-01-24 12:30:00 -0800417 main.Network.switch( SW=switch, OPTION="start" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700418 main.log.info( "Waiting %s seconds for switch up to be discovered" % (
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700419 main.switchSleep ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700420 time.sleep( main.switchSleep )
Devin Lim142b5342017-07-20 15:22:39 -0700421 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700422 main.FALSE,
423 kwargs={ 'numoswitch': switches,
424 'numolink': links },
425 attempts=10,
426 sleep=main.switchSleep )
427 utilities.assert_equals( expect=main.TRUE, actual=topology,
428 onpass="Switch recovery successful",
429 onfail="Failed to recover switch?" )
430
431 @staticmethod
432 def cleanup( main ):
433 """
434 Stop Onos-cluster.
435 Stops Mininet
436 Copies ONOS log
437 """
Devin Lim58046fa2017-07-05 16:55:00 -0700438 try:
439 from tests.dependencies.utils import Utils
440 except ImportError:
441 main.log.error( "Utils not found exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -0700442 main.cleanAndExit()
Devin Lim58046fa2017-07-05 16:55:00 -0700443 try:
Devin Lim142b5342017-07-20 15:22:39 -0700444 main.utils
Devin Lim58046fa2017-07-05 16:55:00 -0700445 except ( NameError, AttributeError ):
Devin Lim142b5342017-07-20 15:22:39 -0700446 main.utils = Utils()
Devin Lim58046fa2017-07-05 16:55:00 -0700447
448 main.utils.mininetCleanup( main.Mininet1 )
449
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900450 main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True )
Devin Lim58046fa2017-07-05 16:55:00 -0700451
Devin Lim142b5342017-07-20 15:22:39 -0700452 for ctrl in main.Cluster.active():
453 main.ONOSbench.onosStop( ctrl.ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700454
455 @staticmethod
456 def killOnos( main, nodes, switches, links, expNodes ):
457 """
458 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
459 switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
460 Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
461 """
462 main.step( "Killing ONOS instance" )
Pier3b58c652016-09-26 12:03:31 -0700463
Jon Hall1efcb3f2016-08-23 13:42:15 -0700464 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700465 killResult = main.ONOSbench.onosDie( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700466 utilities.assert_equals( expect=main.TRUE, actual=killResult,
467 onpass="ONOS instance Killed",
468 onfail="Error killing ONOS instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700469 main.Cluster.runningNodes[ i ].active = False
Jon Hall1efcb3f2016-08-23 13:42:15 -0700470 time.sleep( 12 )
Pier3b58c652016-09-26 12:03:31 -0700471
Devin Lim142b5342017-07-20 15:22:39 -0700472 if len( nodes ) < main.Cluster.numCtrls:
Pier3b58c652016-09-26 12:03:31 -0700473
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700474 nodeResults = utilities.retry( main.Cluster.nodesCheck,
Pier3b58c652016-09-26 12:03:31 -0700475 False,
Pier3b58c652016-09-26 12:03:31 -0700476 attempts=5,
477 sleep=10 )
478 utilities.assert_equals( expect=True, actual=nodeResults,
479 onpass="Nodes check successful",
480 onfail="Nodes check NOT successful" )
481
482 if not nodeResults:
483 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700484 ctrl = main.Cluster.runningNodes[ i ]
Pier3b58c652016-09-26 12:03:31 -0700485 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700486 ctrl.name,
487 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Pier3b58c652016-09-26 12:03:31 -0700488 main.log.error( "Failed to kill ONOS, stopping test" )
Devin Lim44075962017-08-11 10:56:37 -0700489 main.cleanAndExit()
Pier3b58c652016-09-26 12:03:31 -0700490
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900491 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700492 main.FALSE,
493 kwargs={ 'numoswitch': switches,
494 'numolink': links,
495 'numoctrl': expNodes },
496 attempts=10,
497 sleep=12 )
498 utilities.assert_equals( expect=main.TRUE, actual=topology,
499 onpass="ONOS Instance down successful",
500 onfail="Failed to turn off ONOS Instance" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700501
502 @staticmethod
503 def recoverOnos( main, nodes, switches, links, expNodes ):
504 """
505 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
506 switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
507 Recover an ONOS instance and verify the ONOS cluster can see the proper change
508 """
509 main.step( "Recovering ONOS instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700510 [ main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) for i in nodes ]
Jon Hall1efcb3f2016-08-23 13:42:15 -0700511 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700512 isUp = main.ONOSbench.isup( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700513 utilities.assert_equals( expect=main.TRUE, actual=isUp,
514 onpass="ONOS service is ready",
515 onfail="ONOS service did not start properly" )
516 for i in nodes:
517 main.step( "Checking if ONOS CLI is ready" )
Devin Lim142b5342017-07-20 15:22:39 -0700518 ctrl = main.Cluster.runningNodes[ i ]
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900519 # ctrl.CLI.startCellCli()
Devin Lim142b5342017-07-20 15:22:39 -0700520 cliResult = ctrl.CLI.startOnosCli( ctrl.ipAddress,
521 commandlineTimeout=60,
522 onosStartTimeout=100 )
523 ctrl.active = True
Jon Hall1efcb3f2016-08-23 13:42:15 -0700524 utilities.assert_equals( expect=main.TRUE,
525 actual=cliResult,
526 onpass="ONOS CLI is ready",
527 onfail="ONOS CLI is not ready" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700528
Pier3b58c652016-09-26 12:03:31 -0700529 main.step( "Checking ONOS nodes" )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700530 nodeResults = utilities.retry( main.Cluster.nodesCheck,
Pier3b58c652016-09-26 12:03:31 -0700531 False,
Pier3b58c652016-09-26 12:03:31 -0700532 attempts=5,
533 sleep=10 )
534 utilities.assert_equals( expect=True, actual=nodeResults,
535 onpass="Nodes check successful",
536 onfail="Nodes check NOT successful" )
537
538 if not nodeResults:
539 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -0700540 ctrl = main.Cluster.runningNodes[ i ]
Pier3b58c652016-09-26 12:03:31 -0700541 main.log.debug( "{} components not ACTIVE: \n{}".format(
Devin Lim142b5342017-07-20 15:22:39 -0700542 ctrl.name,
543 ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
Pier3b58c652016-09-26 12:03:31 -0700544 main.log.error( "Failed to start ONOS, stopping test" )
Devin Lim44075962017-08-11 10:56:37 -0700545 main.cleanAndExit()
Pier3b58c652016-09-26 12:03:31 -0700546
Devin Lim142b5342017-07-20 15:22:39 -0700547 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700548 main.FALSE,
549 kwargs={ 'numoswitch': switches,
550 'numolink': links,
551 'numoctrl': expNodes },
552 attempts=10,
553 sleep=12 )
554 utilities.assert_equals( expect=main.TRUE, actual=topology,
555 onpass="ONOS Instance down successful",
556 onfail="Failed to turn off ONOS Instance" )
Devin Lim142b5342017-07-20 15:22:39 -0700557 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
558 main.FALSE,
559 attempts=10,
560 sleep=12 )
561 if ready:
562 ready = main.TRUE
563 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700564 onpass="ONOS summary command succeded",
565 onfail="ONOS summary command failed" )
566 if not ready:
567 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700568 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700569
570 @staticmethod
571 def addHostCfg( main ):
572 """
573 Adds Host Configuration to ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700574 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700575 """
576 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700577 hostCfg = {}
You Wangac02b142018-01-26 14:57:28 -0800578 with open( main.configPath + "/json/extra.json" ) as template:
Jon Hall1efcb3f2016-08-23 13:42:15 -0700579 hostCfg = json.load( template )
580 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'in1' ]
581 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700582 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700583 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
584 subjectClass="hosts",
585 subjectKey=urllib.quote( mac,
586 safe='' ),
587 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700588 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'out1' ]
589 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700590 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700591 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
592 subjectClass="hosts",
593 subjectKey=urllib.quote( mac,
594 safe='' ),
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700595 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700596 main.pingChart.update( { 'vlan1': { "expect": "True",
597 "hosts": [ "olt1", "vsg1" ] } } )
598 main.pingChart[ 'vlan5' ][ 'expect' ] = 0
599 main.pingChart[ 'vlan10' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700600 ports = "[%s,%s]" % ( 5, 6 )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700601 cfg = '{"of:0000000000000001":[{"vlan":1,"ports":%s,"name":"OLT 1"}]}' % ports
Devin Lim142b5342017-07-20 15:22:39 -0700602 main.Cluster.active( 0 ).REST.setNetCfg( json.loads( cfg ),
603 subjectClass="apps",
604 subjectKey="org.onosproject.segmentrouting",
605 configKey="xconnect" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700606
607 @staticmethod
608 def delHostCfg( main ):
609 """
610 Removest Host Configuration from ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700611 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700612 """
613 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700614 hostCfg = {}
You Wangac02b142018-01-26 14:57:28 -0800615 with open( main.configPath + "/json/extra.json" ) as template:
Jon Hall1efcb3f2016-08-23 13:42:15 -0700616 hostCfg = json.load( template )
617 main.step( "Removing host configuration" )
618 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700619 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700620 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
621 subjectKey=urllib.quote(
622 mac,
623 safe='' ),
624 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700625 main.step( "Removing configuration" )
626 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700627 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -0700628 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
629 subjectKey=urllib.quote(
630 mac,
631 safe='' ),
632 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700633 main.step( "Removing vlan configuration" )
634 main.pingChart[ 'vlan1' ][ 'expect' ] = 0
Devin Lim142b5342017-07-20 15:22:39 -0700635 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="apps",
636 subjectKey="org.onosproject.segmentrouting",
637 configKey="xconnect" )