blob: 87e1becfd2c956bbeff91658309dbbf50431df4b [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2016 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jon Hall1efcb3f2016-08-23 13:42:15 -070021import os
Jon Hall1efcb3f2016-08-23 13:42:15 -070022import time
23import json
24import urllib
Andreas Pantelopoulosdf5061f2018-05-15 11:46:59 -070025import re
Jon Hallf6aeda22020-07-28 09:12:56 -070026import pexpect
Jon Hall1efcb3f2016-08-23 13:42:15 -070027from core import utilities
28
29
30class Testcaselib:
Pierfb719b12016-09-19 14:51:44 -070031
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070032 useSSH = True
Pierfb719b12016-09-19 14:51:44 -070033
Jon Hall1efcb3f2016-08-23 13:42:15 -070034 @staticmethod
35 def initTest( main ):
36 """
37 - Construct tests variables
38 - GIT ( optional )
39 - Checkout ONOS master branch
40 - Pull latest ONOS code
41 - Building ONOS ( optional )
42 - Install ONOS package
43 - Build ONOS package
44 """
Devin Lim58046fa2017-07-05 16:55:00 -070045 try:
46 from tests.dependencies.ONOSSetup import ONOSSetup
47 main.testSetUp = ONOSSetup()
48 except ImportError:
49 main.log.error( "ONOSSetup not found. exiting the test" )
Devin Lim44075962017-08-11 10:56:37 -070050 main.cleanAndExit()
You Wangd5873482018-01-24 12:30:00 -080051 from tests.dependencies.Network import Network
52 main.Network = Network()
Jon Hall43060f62020-06-23 13:13:33 -070053 main.physicalNet = False
Devin Lim0c972b72018-02-08 14:53:59 -080054 main.testSetUp.envSetupDescription( False )
Devin Lim58046fa2017-07-05 16:55:00 -070055 stepResult = main.FALSE
56 try:
Devin Lim58046fa2017-07-05 16:55:00 -070057 # Test variables
58 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
59 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
Devin Lim58046fa2017-07-05 16:55:00 -070060 main.path = os.path.dirname( main.testFile )
Devin Lim57221b02018-02-14 15:45:36 -080061 main.useCommonTopo = main.params[ 'DEPENDENCY' ][ 'useCommonTopo' ] == 'True'
62 main.topoPath = main.path + ( "/.." if main.useCommonTopo else "" ) + "/dependencies/"
63 main.useCommonConf = main.params[ 'DEPENDENCY' ][ 'useCommonConf' ] == 'True'
You Wang68568b12019-03-04 11:49:57 -080064 if main.params[ 'DEPENDENCY' ].get( 'useBmv2' ):
65 main.useBmv2 = main.params[ 'DEPENDENCY' ][ 'useBmv2' ] == 'True'
66 else:
67 main.useBmv2 = False
Jon Hall43060f62020-06-23 13:13:33 -070068 if main.useBmv2:
69 main.switchType = main.params[ 'DEPENDENCY' ].get( 'bmv2SwitchType', 'stratum' )
70 else:
71 main.switchType = "ovs"
72
Devin Lim57221b02018-02-14 15:45:36 -080073 main.configPath = main.path + ( "/.." if main.useCommonConf else "" ) + "/dependencies/"
Jon Hallbc1c1c92020-05-27 09:29:30 -070074 main.bmv2Path = "/tools/dev/mininet/"
Devin Lim57221b02018-02-14 15:45:36 -080075 main.forJson = "json/"
76 main.forChart = "chart/"
77 main.forConfig = "conf/"
78 main.forHost = "host/"
You Wang27317572018-03-06 12:13:11 -080079 main.forSwitchFailure = "switchFailure/"
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -080080 main.forLinkFailure = "linkFailure/"
You Wange24d6272018-03-27 21:18:50 -070081 main.forMulticast = "multicast/"
Devin Lim58046fa2017-07-05 16:55:00 -070082 main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
You Wangd87b2312018-01-30 12:47:17 -080083 main.topologyLib = main.params[ 'DEPENDENCY' ][ 'lib' ] if 'lib' in main.params[ 'DEPENDENCY' ] else None
84 main.topologyConf = main.params[ 'DEPENDENCY' ][ 'conf' ] if 'conf' in main.params[ 'DEPENDENCY' ] else None
You Wang68568b12019-03-04 11:49:57 -080085 main.bmv2 = "bmv2.py"
Jon Halldac3eae2020-06-05 12:04:06 -070086 main.stratumRoot = main.params[ 'DEPENDENCY'][ 'stratumRoot'] if 'stratumRoot' in main.params[ 'DEPENDENCY' ] else None
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070087 main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
Devin Lim58046fa2017-07-05 16:55:00 -070088 main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
You Wang5bf49592020-07-08 18:47:46 -070089 main.trellisOar = main.params[ 'DEPENDENCY' ][ 'trellisOar' ]
90 main.t3Oar = main.params[ 'DEPENDENCY' ][ 't3Oar' ] if 't3Oar' in main.params[ 'DEPENDENCY' ] else None
Jon Hall1efcb3f2016-08-23 13:42:15 -070091
Devin Lim0c972b72018-02-08 14:53:59 -080092 stepResult = main.testSetUp.envSetup( False )
Devin Lim58046fa2017-07-05 16:55:00 -070093 except Exception as e:
94 main.testSetUp.envSetupException( e )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -070095
Jon Hallaa1d9b82020-07-30 13:49:42 -070096 main.testSetUp.envSetupConclusion( stepResult )
Jon Hall1efcb3f2016-08-23 13:42:15 -070097
Jon Hall1efcb3f2016-08-23 13:42:15 -070098 @staticmethod
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -080099 def installOnos( main, vlanCfg=True, skipPackage=False, cliSleep=10,
100 parallel=True ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700101 """
102 - Set up cell
103 - Create cell file
104 - Set cell file
105 - Verify cell file
106 - Kill ONOS process
107 - Uninstall ONOS cluster
108 - Verify ONOS start up
109 - Install ONOS cluster
110 - Connect to cli
111 """
112 # main.scale[ 0 ] determines the current number of ONOS controller
Jon Hall43060f62020-06-23 13:13:33 -0700113 try:
114 if main.params.get( 'EXTERNAL_APPS' ):
115 for app, url in main.params[ 'EXTERNAL_APPS' ].iteritems():
116 main.log.info( "Downloading %s app from %s" )
117 main.ONOSbench.onosFetchApp( url )
118 if not main.apps:
119 main.log.error( "App list is empty" )
120 except Exception as e:
121 main.log.debug( e )
122 main.cleanAndExit()
Jon Hall3c910162018-03-07 14:42:16 -0800123 main.log.info( "Cluster size: " + str( main.Cluster.numCtrls ) )
124 main.log.info( "Cluster ips: " + ', '.join( main.Cluster.getIps() ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700125 main.dynamicHosts = [ 'in1', 'out1' ]
You Wanga0f6ff62018-01-11 15:46:30 -0800126 main.testSetUp.ONOSSetUp( main.Cluster, newCell=True, cellName=main.cellName,
Andreas Pantelopoulos90f0b102018-02-01 13:21:45 -0800127 skipPack=skipPackage,
128 useSSH=Testcaselib.useSSH,
Devin Lim0c972b72018-02-08 14:53:59 -0800129 installParallel=parallel, includeCaseDesc=False )
Devin Lim142b5342017-07-20 15:22:39 -0700130 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
You Wang5bf49592020-07-08 18:47:46 -0700131 [ None, main.FALSE ],
You Wang1cdc5f52017-12-19 16:47:51 -0800132 sleep=cliSleep,
Devin Lim142b5342017-07-20 15:22:39 -0700133 attempts=10 )
134 if ready:
135 ready = main.TRUE
136 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700137 onpass="ONOS summary command succeded",
138 onfail="ONOS summary command failed" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700139 if not ready:
140 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700141 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700142
You Wang5bf49592020-07-08 18:47:46 -0700143 # Install segmentrouting and t3 app
144 appInstallResult = main.ONOSbench.onosAppInstall( main.Cluster.runningNodes[0].ipAddress, main.trellisOar)
145 if main.t3Oar:
146 appInstallResult = appInstallResult and main.ONOSbench.onosAppInstall( main.Cluster.runningNodes[0].ipAddress, main.t3Oar)
147 utilities.assert_equals( expect=main.TRUE, actual=appInstallResult,
148 onpass="SR app installation succeded",
149 onfail="SR app installation failed" )
150 if not appInstallResult:
151 main.cleanAndExit()
152
Jon Hall43060f62020-06-23 13:13:33 -0700153 # FIXME: move to somewhere else?
154 switchPrefix = main.params[ 'DEPENDENCY' ].get( 'switchPrefix' )
155 # TODO: Support other pipeconfs/making this configurable
156 if switchPrefix == "tofino":
157 # It seems to take some time for the pipeconfs to be loaded
158 ctrl = main.Cluster.next()
159 for i in range( 120 ):
160 try:
161 main.log.debug( "Checking to see if pipeconfs are loaded" )
162 output = ctrl.CLI.sendline( "pipeconfs" )
163 if "tofino" in output:
164 main.log.debug( "Took around %s seconds for the pipeconf to be loaded" % i )
165 break
166 time.sleep( 1 )
167 except Exception as e:
168 main.log.error( e )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700169
Jon Hall43060f62020-06-23 13:13:33 -0700170 Testcaselib.setOnosLogLevels( main )
171 Testcaselib.setOnosConfig( main )
Jon Halldac3eae2020-06-05 12:04:06 -0700172
Jon Hall1efcb3f2016-08-23 13:42:15 -0700173 @staticmethod
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800174 def loadCount( main ):
Jon Hall43060f62020-06-23 13:13:33 -0700175 with open( "%s/count/%s.count" % ( main.configPath, main.cfgName ) ) as count:
176 main.count = json.load( count )
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800177
178 @staticmethod
Jon Hall43060f62020-06-23 13:13:33 -0700179 def loadJson( main, suffix='' ):
180 with open( "%s%s.json%s" % ( main.configPath + main.forJson,
181 main.cfgName, suffix ) ) as cfg:
Devin Lim57221b02018-02-14 15:45:36 -0800182 main.Cluster.active( 0 ).REST.setNetCfg( json.load( cfg ) )
183
184 @staticmethod
185 def loadChart( main ):
186 try:
187 with open( "%s%s.chart" % ( main.configPath + main.forChart,
188 main.cfgName ) ) as chart:
Jon Hall43060f62020-06-23 13:13:33 -0700189 main.pingChart = json.load( chart )
Devin Lim57221b02018-02-14 15:45:36 -0800190 except IOError:
191 main.log.warn( "No chart file found." )
192
193 @staticmethod
194 def loadHost( main ):
195 with open( "%s%s.host" % ( main.configPath + main.forHost,
196 main.cfgName ) ) as host:
197 main.expectedHosts = json.load( host )
198
199 @staticmethod
You Wang27317572018-03-06 12:13:11 -0800200 def loadSwitchFailureChart( main ):
201 with open( "%s%s.switchFailureChart" % ( main.configPath + main.forSwitchFailure,
202 main.cfgName ) ) as sfc:
203 main.switchFailureChart = json.load( sfc )
204
205 @staticmethod
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800206 def loadLinkFailureChart( main ):
207 with open( "%s%s.linkFailureChart" % ( main.configPath + main.forLinkFailure,
You Wange24d6272018-03-27 21:18:50 -0700208 main.cfgName ) ) as lfc:
209 main.linkFailureChart = json.load( lfc )
210
211 @staticmethod
212 def loadMulticastConfig( main ):
213 with open( "%s%s.multicastConfig" % ( main.configPath + main.forMulticast,
214 main.cfgName ) ) as cfg:
215 main.multicastConfig = json.load( cfg )
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800216
217 @staticmethod
Jon Hall1efcb3f2016-08-23 13:42:15 -0700218 def startMininet( main, topology, args="" ):
Jon Hall43060f62020-06-23 13:13:33 -0700219 main.log.info( "Copying mininet topology file to mininet machine" )
You Wangd87b2312018-01-30 12:47:17 -0800220 copyResult = main.ONOSbench.scp( main.Mininet1,
221 main.topoPath + main.topology,
You Wang5da39c82018-04-26 22:55:08 -0700222 main.Mininet1.home + "custom",
You Wangd87b2312018-01-30 12:47:17 -0800223 direction="to" )
224 if main.topologyLib:
225 for lib in main.topologyLib.split(","):
226 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
227 main.topoPath + lib,
You Wang5da39c82018-04-26 22:55:08 -0700228 main.Mininet1.home + "custom",
You Wangd87b2312018-01-30 12:47:17 -0800229 direction="to" )
230 if main.topologyConf:
You Wanga877ea42018-04-05 15:27:40 -0700231 import re
232 controllerIPs = [ ctrl.ipAddress for ctrl in main.Cluster.runningNodes ]
233 index = 0
Jon Hall43060f62020-06-23 13:13:33 -0700234 destDir = "~/"
235 if 'MN_DOCKER' in main.params and main.params['MN_DOCKER']['args']:
236 destDir = "/tmp/mn_conf/"
237 # Try to ensure the destination exists
238 main.log.info( "Create folder for network config files" )
239 handle = main.Mininet1.handle
240 handle.sendline( "mkdir -p %s" % destDir )
241 handle.expect( [ main.Mininet1.prompt, main.Mininet1.dockerPrompt ] )
242 main.log.debug( handle.before + handle.after )
243 # Make sure permissions are correct
244 handle.sendline( "sudo chown %s:%s %s" % ( main.Mininet1.user_name, main.Mininet1.user_name, destDir ) )
245 handle.expect( [ main.Mininet1.prompt, main.Mininet1.dockerPrompt ] )
246 main.log.debug( handle.before + handle.after )
247 handle.sendline( "sudo chmod -R a+rwx %s" % ( destDir ) )
248 handle.expect( [ main.Mininet1.prompt, main.Mininet1.dockerPrompt ] )
249 main.log.debug( handle.before + handle.after )
You Wangd87b2312018-01-30 12:47:17 -0800250 for conf in main.topologyConf.split(","):
You Wanga877ea42018-04-05 15:27:40 -0700251 # Update zebra configurations with correct ONOS instance IP
252 if conf in [ "zebradbgp1.conf", "zebradbgp2.conf" ]:
253 ip = controllerIPs[ index ]
254 index = ( index + 1 ) % len( controllerIPs )
255 with open( main.configPath + main.forConfig + conf ) as f:
256 s = f.read()
257 s = re.sub( r"(fpm connection ip).*(port 2620)", r"\1 " + ip + r" \2", s )
258 with open( main.configPath + main.forConfig + conf, "w" ) as f:
259 f.write( s )
You Wangd87b2312018-01-30 12:47:17 -0800260 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
Devin Lim57221b02018-02-14 15:45:36 -0800261 main.configPath + main.forConfig + conf,
Jon Hall43060f62020-06-23 13:13:33 -0700262 destDir,
You Wangd87b2312018-01-30 12:47:17 -0800263 direction="to" )
You Wang68568b12019-03-04 11:49:57 -0800264 copyResult = copyResult and main.ONOSbench.scp( main.Mininet1,
Jon Hallbc1c1c92020-05-27 09:29:30 -0700265 main.ONOSbench.home + main.bmv2Path + main.bmv2,
You Wang68568b12019-03-04 11:49:57 -0800266 main.Mininet1.home + "custom",
267 direction="to" )
You Wangd87b2312018-01-30 12:47:17 -0800268 stepResult = copyResult
269 utilities.assert_equals( expect=main.TRUE,
270 actual=stepResult,
271 onpass="Successfully copied topo files",
272 onfail="Failed to copy topo files" )
Jon Halldac3eae2020-06-05 12:04:06 -0700273 if main.stratumRoot:
274 main.Mininet1.handle.sendline( "export STRATUM_ROOT=" + str( main.stratumRoot ) )
275 main.Mininet1.handle.expect( main.Mininet1.prompt )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700276 main.step( "Starting Mininet Topology" )
Jonghwan Hyun3731d6a2017-10-19 11:59:31 -0700277 arg = "--onos-ip=%s %s" % (",".join([ctrl.ipAddress for ctrl in main.Cluster.runningNodes]), args)
Jon Hall1efcb3f2016-08-23 13:42:15 -0700278 main.topology = topology
279 topoResult = main.Mininet1.startNet(
You Wang5da39c82018-04-26 22:55:08 -0700280 topoFile=main.Mininet1.home + "custom/" + main.topology, args=arg )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700281 stepResult = topoResult
282 utilities.assert_equals( expect=main.TRUE,
283 actual=stepResult,
284 onpass="Successfully loaded topology",
285 onfail="Failed to load topology" )
286 # Exit if topology did not load properly
287 if not topoResult:
Devin Lim44075962017-08-11 10:56:37 -0700288 main.cleanAndExit()
Jon Hall43060f62020-06-23 13:13:33 -0700289 if main.useBmv2:
290 # Upload the net-cfg file created for each switch
291 filename = "onos-netcfg.json"
292 switchPrefix = main.params[ 'DEPENDENCY' ].get( 'switchPrefix', "bmv2" )
293 for switch in main.Mininet1.getSwitches( switchRegex=r"(StratumBmv2Switch)|(Bmv2Switch)" ).keys():
294 path = "/tmp/mn-stratum/%s/" % switch
295 dstPath = "/tmp/"
296 dstFileName = "%s-onos-netcfg.json" % switch
297 main.ONOSbench1.scp( main.Mininet1,
298 "%s%s" % ( path, filename ),
299 "%s%s" % ( dstPath, dstFileName ),
300 "from" )
301 main.ONOSbench1.handle.sendline( "sudo sed -i 's/localhost/%s/g' %s%s" % ( main.Mininet1.ip_address, dstPath, dstFileName ) )
302 # Configure managementAddress
303 main.ONOSbench1.handle.sendline( "sudo sed -i 's/localhost/%s/g' %s%s" % ( main.Mininet1.ip_address, dstPath, dstFileName ) )
304 main.ONOSbench1.handle.expect( main.ONOSbench1.prompt )
305 main.log.debug( main.ONOSbench1.handle.before + main.ONOSbench1.handle.after )
306 # Configure device id
307 main.ONOSbench1.handle.sendline( "sudo sed -i 's/device:%s/device:%s:%s/g' %s%s" % ( switch, switchPrefix, switch, dstPath, dstFileName ) )
308 main.ONOSbench1.handle.expect( main.ONOSbench1.prompt )
309 main.log.debug( main.ONOSbench1.handle.before + main.ONOSbench1.handle.after )
310 # Configure device name
311 main.ONOSbench1.handle.sendline( "sudo sed -i '/\"basic\"/a\ \"name\": \"%s\",' %s%s" % ( switch, dstPath, dstFileName ) )
312 main.ONOSbench1.handle.expect( main.ONOSbench1.prompt )
313 main.log.debug( main.ONOSbench1.handle.before + main.ONOSbench1.handle.after )
314 main.ONOSbench1.onosNetCfg( main.ONOSserver1.ip_address, dstPath, dstFileName )
315 # Make sure hosts make some noise
316 Testcaselib.discoverHosts( main )
317
318 @staticmethod
319 def discoverHosts( main ):
320 # TODO add option to only select specific hosts
321 if hasattr( main, "Mininet1" ):
322 network = main.Mininet1
323 elif hasattr( main, "NetworkBench" ):
324 network = main.NetworkBench
325 else:
326 main.log.warn( "Could not find component for test network, skipping host discovery" )
327 return
328 network.discoverHosts()
Jon Hall1efcb3f2016-08-23 13:42:15 -0700329
330 @staticmethod
You Wang4cc61912018-08-28 10:10:58 -0700331 def connectToPhysicalNetwork( main ):
You Wang84f981d2018-01-12 16:11:50 -0800332 main.step( "Connecting to physical netowrk" )
Jon Hall43060f62020-06-23 13:13:33 -0700333 main.physicalNet = True
You Wang84f981d2018-01-12 16:11:50 -0800334 topoResult = main.NetworkBench.connectToNet()
335 stepResult = topoResult
336 utilities.assert_equals( expect=main.TRUE,
337 actual=stepResult,
Jon Hall43060f62020-06-23 13:13:33 -0700338 onpass="Successfully connected to topology",
339 onfail="Failed to connect to topology" )
You Wang84f981d2018-01-12 16:11:50 -0800340 # Exit if topology did not load properly
341 if not topoResult:
342 main.cleanAndExit()
343
Jon Hall43060f62020-06-23 13:13:33 -0700344 # Perform any optional setup
345 for switch in main.NetworkBench.switches:
346 if hasattr( switch, "setup" ):
347 switch.setup() # We might not need this
348
You Wang84f981d2018-01-12 16:11:50 -0800349 main.step( "Assign switches to controllers." )
Jon Hall43060f62020-06-23 13:13:33 -0700350 stepResult = main.TRUE
You Wang4cc61912018-08-28 10:10:58 -0700351 switches = main.NetworkBench.getSwitches()
352 pool = []
353 for name in switches.keys():
Jon Hall43060f62020-06-23 13:13:33 -0700354 # NOTE: although this terminology is ovsdb centric, we can use this function for other switches too
355 # e.g. push onos net-cfg for stratum switches
You Wang4cc61912018-08-28 10:10:58 -0700356 thread = main.Thread( target=main.NetworkBench.assignSwController,
357 name="assignSwitchToController",
358 args=[ name, main.Cluster.getIps(), '6653' ] )
359 pool.append( thread )
360 thread.start()
361 for thread in pool:
362 thread.join( 300 )
363 if not thread.result:
364 stepResult = main.FALSE
You Wang84f981d2018-01-12 16:11:50 -0800365 utilities.assert_equals( expect=main.TRUE,
366 actual=stepResult,
367 onpass="Successfully assign switches to controllers",
368 onfail="Failed to assign switches to controllers" )
369
You Wang4cc61912018-08-28 10:10:58 -0700370 # Check devices
371 Testcaselib.checkDevices( main, switches=int( main.params[ 'TOPO' ][ 'switchNum' ] ) )
You Wang4cc61912018-08-28 10:10:58 -0700372 # Connecting to hosts that only have data plane connectivity
373 main.step( "Connecting inband hosts" )
374 stepResult = main.Network.connectInbandHosts()
375 utilities.assert_equals( expect=main.TRUE,
376 actual=stepResult,
377 onpass="Successfully connected inband hosts",
378 onfail="Failed to connect inband hosts" )
Jon Hall43060f62020-06-23 13:13:33 -0700379 Testcaselib.discoverHosts( main )
You Wang4cc61912018-08-28 10:10:58 -0700380
You Wang84f981d2018-01-12 16:11:50 -0800381 @staticmethod
You Wang5df1c6d2018-04-06 18:02:02 -0700382 def saveOnosDiagnostics( main ):
383 """
384 Get onos-diags.tar.gz and save it to the log directory.
385 suffix: suffix string of the file name. E.g. onos-diags-case1.tar.gz
386 """
387 main.log.info( "Collecting onos-diags..." )
388 main.ONOSbench.onosDiagnostics( [ctrl.ipAddress for ctrl in main.Cluster.runningNodes],
389 main.logdir,
390 "-CASE%d" % main.CurrentTestCaseNumber )
391
392 @staticmethod
Devin Lim142b5342017-07-20 15:22:39 -0700393 def config( main, cfgName ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700394 main.spines = []
Piera2a7e1b2016-10-04 11:51:43 -0700395
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700396 main.failures = int( main.params[ 'failures' ] )
397 main.cfgName = cfgName
Piera2a7e1b2016-10-04 11:51:43 -0700398
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700399 if main.cfgName == '2x2':
400 spine = {}
401 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
402 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
403 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700404
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700405 spine = {}
406 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
407 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
408 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700409
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700410 elif main.cfgName == '4x4':
411 spine = {}
412 spine[ 'name' ] = main.params[ 'switches' ][ 'spine1' ]
413 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid1' ]
414 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700415
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700416 spine = {}
417 spine[ 'name' ] = main.params[ 'switches' ][ 'spine2' ]
418 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid2' ]
419 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700420
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700421 spine = {}
422 spine[ 'name' ] = main.params[ 'switches' ][ 'spine3' ]
423 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid3' ]
424 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700425
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700426 spine = {}
427 spine[ 'name' ] = main.params[ 'switches' ][ 'spine4' ]
428 spine[ 'dpid' ] = main.params[ 'switches' ][ 'spinedpid4' ]
429 main.spines.append( spine )
Piera2a7e1b2016-10-04 11:51:43 -0700430
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700431 else:
Piera2a7e1b2016-10-04 11:51:43 -0700432 main.log.error( "Configuration failed!" )
Devin Lim44075962017-08-11 10:56:37 -0700433 main.cleanAndExit()
You Wang27317572018-03-06 12:13:11 -0800434
Andreas Pantelopoulos2eae3242018-03-06 13:47:20 -0800435 @staticmethod
436 def addStaticOnosRoute( main, subnet, intf):
437 """
438 Adds an ONOS static route with the use route-add command.
439 """
Andreas Pantelopoulos2eae3242018-03-06 13:47:20 -0800440 routeResult = main.Cluster.active( 0 ).addStaticRoute(subnet, intf)
441
Piera2a7e1b2016-10-04 11:51:43 -0700442 @staticmethod
You Wangc02f3be2018-05-18 12:14:23 -0700443 def checkGroupsForBuckets( main, deviceId, subnetDict, routingTable=30 ):
Andreas Pantelopoulosdf5061f2018-05-15 11:46:59 -0700444 """
445 Check number of groups for each subnet on device deviceId and matches
446 it with an expected value. subnetDict is a dictionarty containing values
447 of the type "10.0.1.0/24" : 5.
448 """
You Wangc02f3be2018-05-18 12:14:23 -0700449 main.step( "Checking if number of groups for subnets in device {0} is as expected.".format( deviceId ) )
450 groups = main.Cluster.active( 0 ).CLI.getGroups( deviceId, groupType="select" )
451 flows = main.Cluster.active( 0 ).CLI.flows( jsonFormat=False, device=deviceId )
Andreas Pantelopoulosdf5061f2018-05-15 11:46:59 -0700452
You Wangc02f3be2018-05-18 12:14:23 -0700453 result = main.TRUE
454 for subnet, numberInSelect in subnetDict.iteritems():
Andreas Pantelopoulosdf5061f2018-05-15 11:46:59 -0700455 for flow in flows.splitlines():
You Wangc02f3be2018-05-18 12:14:23 -0700456 if "tableId={0}".format( routingTable ) in flow and subnet in flow:
Andreas Pantelopoulosdf5061f2018-05-15 11:46:59 -0700457 # this will match the group id that this flow entry points to, for example :
458 # 0x70000041 in flow entry which contains "deferred=[GROUP:0x70000041], transition=TABLE:60,"
You Wangc02f3be2018-05-18 12:14:23 -0700459 groupId = re.search( r".*GROUP:(0x.*)], transition.*", flow ).groups()[0]
Andreas Pantelopoulosdf5061f2018-05-15 11:46:59 -0700460 count = 0
461 for group in groups.splitlines():
You Wangc02f3be2018-05-18 12:14:23 -0700462 if 'id={0}'.format( groupId ) in group:
Andreas Pantelopoulosdf5061f2018-05-15 11:46:59 -0700463 count += 1
You Wangc02f3be2018-05-18 12:14:23 -0700464 if count - 1 != numberInSelect:
465 result = main.FALSE
466 main.log.warn( "Mismatch in number of buckets of select group, found {0}, expected {1} for subnet {2} on device {3}".format( count - 1, numberInSelect, subnet, deviceId ) )
467 utilities.assert_equals( expect=main.TRUE, actual=result,
468 onpass="All bucket numbers are as expected",
469 onfail="Some bucket numbers are not as expected" )
Andreas Pantelopoulosdf5061f2018-05-15 11:46:59 -0700470
471 @staticmethod
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900472 def checkFlows( main, minFlowCount, tag="", dumpflows=True, sleep=10 ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700473 main.step(
Jon Hall3c910162018-03-07 14:42:16 -0800474 "Check whether the flow count is bigger than %s" % minFlowCount )
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900475 if tag == "":
476 tag = 'CASE%d' % main.CurrentTestCaseNumber
Devin Lim142b5342017-07-20 15:22:39 -0700477 count = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowCount,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700478 main.FALSE,
479 kwargs={ 'min': minFlowCount },
480 attempts=10,
You Wang1cdc5f52017-12-19 16:47:51 -0800481 sleep=sleep )
steven30801eccfe212019-01-24 13:00:42 +0800482 if count == main.FALSE:
483 count = main.Cluster.active( 0 ).CLI.checkFlowCount()
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700484 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700485 expect=True,
steven30801eccfe212019-01-24 13:00:42 +0800486 actual=( count > minFlowCount ),
Jon Hall43060f62020-06-23 13:13:33 -0700487 onpass="Flow count looks correct; found %s, expecting at least %s" % ( count, minFlowCount ),
488 onfail="Flow count looks wrong; found %s, expecting at least %s" % ( count, minFlowCount ) )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700489
490 main.step( "Check whether all flow status are ADDED" )
Devin Lim142b5342017-07-20 15:22:39 -0700491 flowCheck = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowsState,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700492 main.FALSE,
493 kwargs={ 'isPENDING': False },
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800494 attempts=5,
You Wang1cdc5f52017-12-19 16:47:51 -0800495 sleep=sleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700496 utilities.assertEquals(
Jon Hall1efcb3f2016-08-23 13:42:15 -0700497 expect=main.TRUE,
498 actual=flowCheck,
499 onpass="Flow status is correct!",
500 onfail="Flow status is wrong!" )
501 if dumpflows:
Devin Lim142b5342017-07-20 15:22:39 -0700502 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700503 "flows",
504 main.logdir,
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900505 tag + "_FlowsBefore" )
Devin Lim142b5342017-07-20 15:22:39 -0700506 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
Pier50f0bc62016-09-07 17:53:40 -0700507 "groups",
508 main.logdir,
Jonghwan Hyun76a02b72018-01-30 16:40:48 +0900509 tag + "_GroupsBefore" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700510
511 @staticmethod
Pier6a0c4de2018-03-18 16:01:30 -0700512 def checkDevices( main, switches, tag="", sleep=10 ):
513 main.step(
514 "Check whether the switches count is equal to %s" % switches )
515 if tag == "":
516 tag = 'CASE%d' % main.CurrentTestCaseNumber
517 result = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
518 main.FALSE,
519 kwargs={ 'numoswitch': switches},
520 attempts=10,
521 sleep=sleep )
522 utilities.assert_equals( expect=main.TRUE, actual=result,
523 onpass="Device up successful",
524 onfail="Failed to boot up devices?" )
525
526 @staticmethod
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800527 def checkFlowsByDpid( main, dpid, minFlowCount, sleep=10 ):
528 main.step(
Jon Hall43060f62020-06-23 13:13:33 -0700529 "Check whether the flow count of device %s is bigger than %s" % ( dpid, minFlowCount ) )
Jonghwan Hyuncf2345c2018-02-26 11:07:54 -0800530 count = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowAddedCount,
531 main.FALSE,
532 args=( dpid, minFlowCount ),
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800533 attempts=5,
534 sleep=sleep )
steven30801eccfe212019-01-24 13:00:42 +0800535 if count == main.FALSE:
536 count = main.Cluster.active( 0 ).CLI.checkFlowAddedCount( dpid )
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800537 utilities.assertEquals(
Jonghwan Hyuncf2345c2018-02-26 11:07:54 -0800538 expect=True,
539 actual=( count > minFlowCount ),
540 onpass="Flow count looks correct: " + str( count ),
steven30801eccfe212019-01-24 13:00:42 +0800541 onfail="Flow count looks wrong: " + str( count ) )
Jonghwan Hyun98fb40a2018-01-04 16:16:28 -0800542
543 @staticmethod
Jon Hall9677ed32018-04-24 11:16:23 -0700544 def checkFlowEqualityByDpid( main, dpid, flowCount, sleep=10 ):
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800545 main.step(
Jon Hall43060f62020-06-23 13:13:33 -0700546 "Check whether the flow count of device %s is equal to %s" % ( dpid, flowCount ) )
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800547 count = utilities.retry( main.Cluster.active( 0 ).CLI.checkFlowAddedCount,
548 main.FALSE,
Jon Hall9677ed32018-04-24 11:16:23 -0700549 args=( dpid, flowCount, False, 1 ),
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800550 attempts=5,
551 sleep=sleep )
steven30801eccfe212019-01-24 13:00:42 +0800552 if count == main.FALSE:
553 count = main.Cluster.active( 0 ).CLI.checkFlowAddedCount( dpid )
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800554 utilities.assertEquals(
555 expect=True,
steven30801eccfe212019-01-24 13:00:42 +0800556 actual=( count == flowCount ),
Jon Hall9677ed32018-04-24 11:16:23 -0700557 onpass="Flow count looks correct: " + str( count ) ,
558 onfail="Flow count looks wrong. found {}, should be {}.".format( count, flowCount ) )
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800559
560 @staticmethod
561 def checkGroupEqualityByDpid( main, dpid, groupCount, sleep=10):
562 main.step(
Jon Hall43060f62020-06-23 13:13:33 -0700563 "Check whether the group count of device %s is equal to %s" % ( dpid, groupCount ) )
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800564 count = utilities.retry( main.Cluster.active( 0 ).CLI.checkGroupAddedCount,
565 main.FALSE,
566 args=( dpid, groupCount, False, 1),
567 attempts=5,
568 sleep=sleep )
steven30801eccfe212019-01-24 13:00:42 +0800569 if count == main.FALSE:
steven3080123997972019-01-29 17:01:40 +0800570 count = main.Cluster.active( 0 ).CLI.checkGroupAddedCount( dpid )
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800571 utilities.assertEquals(
572 expect=True,
573 actual=( count == groupCount ),
Jon Hall9677ed32018-04-24 11:16:23 -0700574 onpass="Group count looks correct: " + str( count ) ,
575 onfail="Group count looks wrong. found {}, should be {}.".format( count, groupCount ) )
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800576
577 @staticmethod
Jon Hall9677ed32018-04-24 11:16:23 -0700578 def checkFlowsGroupsFromFile( main ):
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800579
580 for dpid, values in main.count.items():
581 flowCount = values["flows"]
582 groupCount = values["groups"]
Jon Hall9677ed32018-04-24 11:16:23 -0700583 main.log.report( "Check flow count for dpid " + str( dpid ) +
584 ", should be " + str( flowCount ) )
585 Testcaselib.checkFlowEqualityByDpid( main, dpid, flowCount )
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800586
Jon Hall9677ed32018-04-24 11:16:23 -0700587 main.log.report( "Check group count for dpid " + str( dpid ) +
588 ", should be " + str( groupCount ) )
589 Testcaselib.checkGroupEqualityByDpid( main, dpid, groupCount )
Andreas Pantelopoulos9173d442018-03-01 17:07:37 -0800590
591 return
592
593 @staticmethod
Jon Halla604fd42018-05-04 14:27:27 -0700594 def pingAll( main, tag="", dumpflows=True, acceptableFailed=0, basedOnIp=False,
595 sleep=10, retryAttempts=1, skipOnFail=False ):
You Wangf19d9f42018-02-23 16:34:19 -0800596 '''
You Wangba231e72018-03-01 13:18:21 -0800597 Verify connectivity between hosts according to the ping chart
598 acceptableFailed: max number of acceptable failed pings.
You Wangf19d9f42018-02-23 16:34:19 -0800599 basedOnIp: if True, run ping or ping6 based on suffix of host names
Jonghwan Hyun812c70f2018-02-16 16:33:16 -0800600 retryAttempts: the number of retry ping. Only works for IPv4 hosts.
You Wangf19d9f42018-02-23 16:34:19 -0800601 '''
You Wangba231e72018-03-01 13:18:21 -0800602 main.log.report( "Check host connectivity" )
603 main.log.debug( "Ping chart: %s" % main.pingChart )
Andreas Pantelopoulosf6ed5012018-02-08 21:26:01 -0800604 if tag == "":
605 tag = 'CASE%d' % main.CurrentTestCaseNumber
606 for entry in main.pingChart.itervalues():
You Wangba231e72018-03-01 13:18:21 -0800607 main.log.debug( "Entry in ping chart: %s" % entry )
608 expect = entry[ 'expect' ]
609 if expect == "Unidirectional":
610 # Verify ping from each src host to each dst host
611 src = entry[ 'src' ]
612 dst = entry[ 'dst' ]
613 expect = main.TRUE
614 main.step( "Verify unidirectional connectivity from %s to %s with tag %s" % ( str( src ), str( dst ), tag ) )
615 if basedOnIp:
616 if ("v4" in src[0]):
617 pa = main.Network.pingallHostsUnidirectional( src, dst, acceptableFailed=acceptableFailed )
618 utilities.assert_equals( expect=expect, actual=pa,
619 onpass="IPv4 connectivity successfully tested",
620 onfail="IPv4 connectivity failed" )
621 if ("v6" in src[0]):
622 pa = main.Network.pingallHostsUnidirectional( src, dst, ipv6=True, acceptableFailed=acceptableFailed )
623 utilities.assert_equals( expect=expect, actual=pa,
624 onpass="IPv6 connectivity successfully tested",
625 onfail="IPv6 connectivity failed" )
Jon Hall43060f62020-06-23 13:13:33 -0700626 elif main.physicalNet:
627 pa = main.NetworkBench.pingallHostsUnidirectional( src, dst, acceptableFailed=acceptableFailed, useScapy=True )
628 utilities.assert_equals( expect=expect, actual=pa,
629 onpass="IP connectivity successfully tested",
630 onfail="IP connectivity failed" )
631
You Wangba231e72018-03-01 13:18:21 -0800632 else:
633 pa = main.Network.pingallHostsUnidirectional( src, dst, acceptableFailed=acceptableFailed )
634 utilities.assert_equals( expect=expect, actual=pa,
635 onpass="IP connectivity successfully tested",
636 onfail="IP connectivity failed" )
637 else:
638 # Verify ping between each host pair
639 hosts = entry[ 'hosts' ]
640 try:
641 expect = main.TRUE if str(expect).lower() == 'true' else main.FALSE
642 except:
643 expect = main.FALSE
644 main.step( "Verify full connectivity for %s with tag %s" % ( str( hosts ), tag ) )
645 if basedOnIp:
646 if ("v4" in hosts[0]):
Jonghwan Hyun812c70f2018-02-16 16:33:16 -0800647 pa = utilities.retry( main.Network.pingallHosts,
648 main.FALSE if expect else main.TRUE,
Jon Hall43060f62020-06-23 13:13:33 -0700649 args=(hosts, ),
650 kwargs={ 'ipv6': False },
Jonghwan Hyun812c70f2018-02-16 16:33:16 -0800651 attempts=retryAttempts,
652 sleep=sleep )
You Wangba231e72018-03-01 13:18:21 -0800653 utilities.assert_equals( expect=expect, actual=pa,
654 onpass="IPv4 connectivity successfully tested",
655 onfail="IPv4 connectivity failed" )
656 if ("v6" in hosts[0]):
657 pa = main.Network.pingIpv6Hosts( hosts, acceptableFailed=acceptableFailed )
658 utilities.assert_equals( expect=expect, actual=pa,
659 onpass="IPv6 connectivity successfully tested",
660 onfail="IPv6 connectivity failed" )
Jon Hall43060f62020-06-23 13:13:33 -0700661 elif main.physicalNet:
662 pa = main.Network.pingallHosts( hosts, ipv6=True, useScapy=True )
663 utilities.assert_equals( expect=expect, actual=pa,
664 onpass="IP connectivity successfully tested",
665 onfail="IP connectivity failed" )
You Wangba231e72018-03-01 13:18:21 -0800666 else:
You Wangf19d9f42018-02-23 16:34:19 -0800667 pa = main.Network.pingallHosts( hosts )
668 utilities.assert_equals( expect=expect, actual=pa,
You Wangba231e72018-03-01 13:18:21 -0800669 onpass="IP connectivity successfully tested",
670 onfail="IP connectivity failed" )
Jon Hall43060f62020-06-23 13:13:33 -0700671 if pa != expect:
You Wang5df1c6d2018-04-06 18:02:02 -0700672 Testcaselib.saveOnosDiagnostics( main )
Jon Hall43060f62020-06-23 13:13:33 -0700673 if skipOnFail and pa != expect:
You Wang24ad2f52018-04-10 10:47:12 -0700674 Testcaselib.cleanup( main, copyKarafLog=False )
You Wang5df1c6d2018-04-06 18:02:02 -0700675 main.skipCase()
Andreas Pantelopoulosf6ed5012018-02-08 21:26:01 -0800676
677 if dumpflows:
678 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
679 "flows",
680 main.logdir,
681 tag + "_FlowsOn" )
682 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress,
683 "groups",
684 main.logdir,
685 tag + "_GroupsOn" )
686
687 @staticmethod
Jon Halla604fd42018-05-04 14:27:27 -0700688 def killLink( main, end1, end2, switches, links, sleep=None ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700689 """
690 end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
691 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
692 Kill a link and verify ONOS can see the proper link change
693 """
Jon Halla604fd42018-05-04 14:27:27 -0700694 if sleep is None:
695 sleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
696 else:
697 sleep = float( sleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700698 main.step( "Kill link between %s and %s" % ( end1, end2 ) )
Jon Halla604fd42018-05-04 14:27:27 -0700699 linkDown = main.Network.link( END1=end1, END2=end2, OPTION="down" )
700 linkDown = linkDown and main.Network.link( END2=end1, END1=end2, OPTION="down" )
701 # TODO: Can remove this, since in the retry we will wait anyways if topology is incorrect
Jon Hall1efcb3f2016-08-23 13:42:15 -0700702 main.log.info(
Jon Halla604fd42018-05-04 14:27:27 -0700703 "Waiting %s seconds for link down to be discovered" % sleep )
704 time.sleep( sleep )
Devin Lim142b5342017-07-20 15:22:39 -0700705 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700706 main.FALSE,
707 kwargs={ 'numoswitch': switches,
708 'numolink': links },
709 attempts=10,
Jon Halla604fd42018-05-04 14:27:27 -0700710 sleep=sleep )
711 result = topology and linkDown
Jon Hall1efcb3f2016-08-23 13:42:15 -0700712 utilities.assert_equals( expect=main.TRUE, actual=result,
713 onpass="Link down successful",
714 onfail="Failed to turn off link?" )
715
716 @staticmethod
Jon Halla604fd42018-05-04 14:27:27 -0700717 def killLinkBatch( main, links, linksAfter, switches, sleep=None ):
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800718 """
719 links = list of links (src, dst) to bring down.
720 """
721
722 main.step("Killing a batch of links {0}".format(links))
Jon Halla604fd42018-05-04 14:27:27 -0700723 if sleep is None:
724 sleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
725 else:
726 sleep = float( sleep )
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800727
728 for end1, end2 in links:
729 main.Network.link( END1=end1, END2=end2, OPTION="down")
730 main.Network.link( END1=end2, END2=end1, OPTION="down")
731
Jon Halla604fd42018-05-04 14:27:27 -0700732 # TODO: Can remove this, since in the retry we will wait anyways if topology is incorrect
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800733 main.log.info(
Jon Halla604fd42018-05-04 14:27:27 -0700734 "Waiting %s seconds for links down to be discovered" % sleep )
735 time.sleep( sleep )
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800736
737 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
738 main.FALSE,
739 kwargs={ 'numoswitch': switches,
740 'numolink': linksAfter },
741 attempts=10,
Jon Halla604fd42018-05-04 14:27:27 -0700742 sleep=sleep )
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800743
You Wang2854bce2018-03-30 10:15:32 -0700744 utilities.assert_equals( expect=main.TRUE, actual=topology,
745 onpass="Link batch down successful",
746 onfail="Link batch down failed" )
747
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800748 @staticmethod
Jon Halla604fd42018-05-04 14:27:27 -0700749 def restoreLinkBatch( main, links, linksAfter, switches, sleep=None ):
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800750 """
751 links = list of link (src, dst) to bring up again.
752 """
753
754 main.step("Restoring a batch of links {0}".format(links))
Jon Halla604fd42018-05-04 14:27:27 -0700755 if sleep is None:
756 sleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
757 else:
758 sleep = float( sleep )
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800759
760 for end1, end2 in links:
761 main.Network.link( END1=end1, END2=end2, OPTION="up")
762 main.Network.link( END1=end2, END2=end1, OPTION="up")
763
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800764 main.log.info(
Jon Halla604fd42018-05-04 14:27:27 -0700765 "Waiting %s seconds for links up to be discovered" % sleep )
766 time.sleep( sleep )
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800767
768 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
769 main.FALSE,
770 kwargs={ 'numoswitch': switches,
771 'numolink': linksAfter },
772 attempts=10,
Jon Halla604fd42018-05-04 14:27:27 -0700773 sleep=sleep )
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800774
You Wang2854bce2018-03-30 10:15:32 -0700775 utilities.assert_equals( expect=main.TRUE, actual=topology,
776 onpass="Link batch up successful",
777 onfail="Link batch up failed" )
778
Andreas Pantelopoulosfab6bf32018-03-06 18:56:35 -0800779 @staticmethod
You Wang85747762018-05-11 15:51:50 -0700780 def disablePortBatch( main, ports, switches=None, links=None, sleep=None ):
781 """
782 Disable a list of switch ports using 'portstate' and verify ONOS can see the proper link change
783 ports: a list of ports to disable ex. [ [ "of:0000000000000001", 1 ] ]
784 switches, links: number of expected switches and links after link change, ex.: '4', '6'
785 """
786 if sleep is None:
787 sleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
788 else:
789 sleep = float( sleep )
790 main.step( "Disable a batch of ports" )
791 for dpid, port in ports:
792 main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="disable" )
793 main.log.info( "Waiting {} seconds for port down to be discovered".format( sleep ) )
794 time.sleep( sleep )
795 if switches and links:
796 result = main.Cluster.active( 0 ).CLI.checkStatus( numoswitch=switches,
797 numolink=links )
798 utilities.assert_equals( expect=main.TRUE, actual=result,
799 onpass="Port down successful",
800 onfail="Port down failed" )
801
802 @staticmethod
803 def enablePortBatch( main, ports, switches, links, sleep=None ):
804 """
805 Enable a list of switch ports using 'portstate' and verify ONOS can see the proper link change
806 ports: a list of ports to enable ex. [ [ "of:0000000000000001", 1 ] ]
807 switches, links: number of expected switches and links after link change, ex.: '4', '6'
808 """
809 if sleep is None:
810 sleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
811 else:
812 sleep = float( sleep )
813 main.step( "Enable a batch of ports" )
814 for dpid, port in ports:
815 main.Cluster.active( 0 ).CLI.portstate( dpid=dpid, port=port, state="enable" )
816 main.log.info( "Waiting {} seconds for port up to be discovered".format( sleep ) )
817 time.sleep( sleep )
818 if switches and links:
819 result = main.Cluster.active( 0 ).CLI.checkStatus( numoswitch=switches,
820 numolink=links )
821 utilities.assert_equals( expect=main.TRUE, actual=result,
822 onpass="Port up successful",
823 onfail="Port up failed" )
824
825 @staticmethod
You Wangc02d8352018-04-17 16:42:10 -0700826 def restoreLink( main, end1, end2, switches, links,
Jon Halla604fd42018-05-04 14:27:27 -0700827 portUp=False, dpid1='', dpid2='', port1='', port2='', sleep=None ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700828 """
829 Params:
830 end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
You Wangc02d8352018-04-17 16:42:10 -0700831 portUp: enable portstate after restoring link
Jon Hall1efcb3f2016-08-23 13:42:15 -0700832 dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
833 port1, port2: respective port of the end switches that connects to the link, ex.:'1'
834 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
835 Kill a link and verify ONOS can see the proper link change
836 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700837 main.step( "Restore link between %s and %s" % ( end1, end2 ) )
Jon Halla604fd42018-05-04 14:27:27 -0700838 if sleep is None:
839 sleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
840 else:
841 sleep = float( sleep )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700842 result = False
843 count = 0
844 while True:
845 count += 1
Jon Halla604fd42018-05-04 14:27:27 -0700846 ctrl = main.Cluster.next()
You Wangd5873482018-01-24 12:30:00 -0800847 main.Network.link( END1=end1, END2=end2, OPTION="up" )
848 main.Network.link( END2=end1, END1=end2, OPTION="up" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700849 main.log.info(
Jon Halla604fd42018-05-04 14:27:27 -0700850 "Waiting %s seconds for link up to be discovered" % sleep )
851 time.sleep( sleep )
Pierfb719b12016-09-19 14:51:44 -0700852
You Wangc02d8352018-04-17 16:42:10 -0700853 if portUp:
854 ctrl.CLI.portstate( dpid=dpid1, port=port1, state='Enable' )
855 ctrl.CLI.portstate( dpid=dpid2, port=port2, state='Enable' )
Jon Hall43060f62020-06-23 13:13:33 -0700856 main.log.info(
857 "Waiting %s seconds for link up to be discovered" % sleep )
Jon Halla604fd42018-05-04 14:27:27 -0700858 time.sleep( sleep )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700859
Jon Halla604fd42018-05-04 14:27:27 -0700860 result = ctrl.CLI.checkStatus( numoswitch=switches,
861 numolink=links )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700862 if count > 5 or result:
863 break
864 utilities.assert_equals( expect=main.TRUE, actual=result,
865 onpass="Link up successful",
866 onfail="Failed to bring link up" )
867
868 @staticmethod
Jon Halla604fd42018-05-04 14:27:27 -0700869 def killSwitch( main, switch, switches, links, sleep=None ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700870 """
871 Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
872 Completely kill a switch and verify ONOS can see the proper change
873 """
Jon Halla604fd42018-05-04 14:27:27 -0700874 if sleep is None:
875 sleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
876 else:
877 sleep = float( sleep )
You Wangc02d8352018-04-17 16:42:10 -0700878 switch = switch if isinstance( switch, list ) else [ switch ]
879 main.step( "Kill " + str( switch ) )
880 for s in switch:
881 main.log.info( "Stopping " + s )
882 main.Network.switch( SW=s, OPTION="stop" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700883 # todo make this repeatable
Jon Halla604fd42018-05-04 14:27:27 -0700884
885 # TODO: Can remove this, since in the retry we will wait anyways if topology is incorrect
Jon Hall1efcb3f2016-08-23 13:42:15 -0700886 main.log.info( "Waiting %s seconds for switch down to be discovered" % (
Jon Halla604fd42018-05-04 14:27:27 -0700887 sleep ) )
888 time.sleep( sleep )
Devin Lim142b5342017-07-20 15:22:39 -0700889 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700890 main.FALSE,
891 kwargs={ 'numoswitch': switches,
892 'numolink': links },
893 attempts=10,
Jon Halla604fd42018-05-04 14:27:27 -0700894 sleep=sleep )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700895 utilities.assert_equals( expect=main.TRUE, actual=topology,
896 onpass="Kill switch successful",
897 onfail="Failed to kill switch?" )
898
899 @staticmethod
Jon Halla604fd42018-05-04 14:27:27 -0700900 def recoverSwitch( main, switch, switches, links, rediscoverHosts=False, hostsToDiscover=[], sleep=None ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700901 """
902 Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
903 Recover a switch and verify ONOS can see the proper change
904 """
Jon Halla604fd42018-05-04 14:27:27 -0700905 if sleep is None:
906 sleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
907 else:
908 sleep = float( sleep )
909 # TODO make this repeatable
You Wangc02d8352018-04-17 16:42:10 -0700910 switch = switch if isinstance( switch, list ) else [ switch ]
911 main.step( "Recovering " + str( switch ) )
912 for s in switch:
913 main.log.info( "Starting " + s )
914 main.Network.switch( SW=s, OPTION="start" )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700915 main.log.info( "Waiting %s seconds for switch up to be discovered" % (
Jon Halla604fd42018-05-04 14:27:27 -0700916 sleep ) )
917 time.sleep( sleep )
Andreas Pantelopoulos74c7ff22018-05-01 15:42:02 -0700918 if rediscoverHosts:
You Wang48381752018-05-07 13:50:57 -0700919 main.Network.discoverHosts( hostList=hostsToDiscover )
Andreas Pantelopoulos74c7ff22018-05-01 15:42:02 -0700920 main.log.info( "Waiting %s seconds for hosts to get re-discovered" % (
Jon Halla604fd42018-05-04 14:27:27 -0700921 sleep ) )
922 time.sleep( sleep )
Andreas Pantelopoulos74c7ff22018-05-01 15:42:02 -0700923
Devin Lim142b5342017-07-20 15:22:39 -0700924 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
Jon Hall1efcb3f2016-08-23 13:42:15 -0700925 main.FALSE,
926 kwargs={ 'numoswitch': switches,
927 'numolink': links },
928 attempts=10,
Jon Halla604fd42018-05-04 14:27:27 -0700929 sleep=sleep )
Jon Hall1efcb3f2016-08-23 13:42:15 -0700930 utilities.assert_equals( expect=main.TRUE, actual=topology,
931 onpass="Switch recovery successful",
932 onfail="Failed to recover switch?" )
933
Jonghwan Hyun25c98a62018-05-04 13:59:09 -0700934 @staticmethod
You Wang0f745de2018-07-27 15:49:22 -0700935 def killRouter( main, router, sleep=None ):
936 """
937 Kill bgpd process on a quagga router
938 router: name of the router to be killed. E.g. "bgp1"
939 """
940 sleep = float( sleep )
941 main.step( "Kill " + str( router ) )
942 if hasattr( main, 'Mininet1' ):
943 main.Mininet1.handle.sendline( "px {}.stopProtocols()".format( router ) )
944 main.Mininet1.handle.expect( "mininet>" )
945 else:
946 # TODO: support killing router in physical network
947 pass
948 main.log.info( "Waiting %s seconds for router down to be discovered" % ( sleep ) )
949 time.sleep( sleep )
950
951 @staticmethod
952 def recoverRouter( main, router, sleep=None ):
953 """
954 Restart bgpd process on a quagga router
955 router: name of the router to be recovered. E.g. "bgp1"
956 """
957 sleep = float( sleep )
958 main.step( "Recovering " + str( router ) )
959 if hasattr( main, 'Mininet1' ):
960 main.Mininet1.handle.sendline( "px {}.startProtocols()".format( router ) )
961 main.Mininet1.handle.expect( "mininet>" )
962 else:
963 # TODO: support recovering router in physical network
964 pass
965 main.log.info( "Waiting %s seconds for router up to be discovered" % ( sleep ) )
966 time.sleep( sleep )
967
968 @staticmethod
You Wang5da39c82018-04-26 22:55:08 -0700969 def cleanup( main, copyKarafLog=True, removeHostComponent=False ):
Jon Hall1efcb3f2016-08-23 13:42:15 -0700970 """
971 Stop Onos-cluster.
972 Stops Mininet
973 Copies ONOS log
974 """
You Wang4cc61912018-08-28 10:10:58 -0700975 from tests.dependencies.utils import Utils
976 main.utils = Utils()
Jon Halldac3eae2020-06-05 12:04:06 -0700977 for ctrl in main.Cluster.active():
978 ctrl.CLI.log( "\"Ending Test - Shutting down ONOS and Network\"", level="INFO" )
You Wang4cc61912018-08-28 10:10:58 -0700979 # Clean up scapy hosts
You Wange24d6272018-03-27 21:18:50 -0700980 if hasattr( main, "scapyHosts" ):
981 scapyResult = main.TRUE
982 for host in main.scapyHosts:
983 scapyResult = host.stopScapy() and scapyResult
984 main.log.info( "Stopped Scapy Host: {0}".format( host.name ) )
985 for host in main.scapyHosts:
You Wang4cc61912018-08-28 10:10:58 -0700986 if hasattr( main, 'Mininet1' ):
987 scapyResult = main.Scapy.removeHostComponent( host.name ) and scapyResult
988 else:
989 scapyResult = main.Network.removeHostComponent( host.name ) and scapyResult
You Wange24d6272018-03-27 21:18:50 -0700990 main.log.info( "Removed Scapy Host Component: {0}".format( host.name ) )
991 main.scapyHosts = []
992
You Wang5da39c82018-04-26 22:55:08 -0700993 if removeHostComponent:
994 for host in main.internalIpv4Hosts + main.internalIpv6Hosts + main.externalIpv4Hosts + main.externalIpv6Hosts:
995 if hasattr( main, host ):
You Wang4cc61912018-08-28 10:10:58 -0700996 if hasattr( main, 'Mininet1' ):
997 pass
998 else:
999 getattr( main, host ).disconnectInband()
You Wang5da39c82018-04-26 22:55:08 -07001000 main.Network.removeHostComponent( host )
1001
You Wang5df1c6d2018-04-06 18:02:02 -07001002 if hasattr( main, 'Mininet1' ):
Pier6a0c4de2018-03-18 16:01:30 -07001003 main.utils.mininetCleanup( main.Mininet1 )
You Wang4cc61912018-08-28 10:10:58 -07001004 else:
1005 main.Network.disconnectInbandHosts()
1006 main.Network.disconnectFromNet()
Devin Lim58046fa2017-07-05 16:55:00 -07001007
You Wang5df1c6d2018-04-06 18:02:02 -07001008 if copyKarafLog:
1009 main.utils.copyKarafLog( "CASE%d" % main.CurrentTestCaseNumber, before=True, includeCaseDesc=False )
Devin Lim58046fa2017-07-05 16:55:00 -07001010
Devin Lim142b5342017-07-20 15:22:39 -07001011 for ctrl in main.Cluster.active():
1012 main.ONOSbench.onosStop( ctrl.ipAddress )
Jon Hall43060f62020-06-23 13:13:33 -07001013 Testcaselib.mnDockerTeardown( main )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001014
1015 @staticmethod
Jon Halla604fd42018-05-04 14:27:27 -07001016 def verifyNodes( main ):
1017 """
1018 Verifies Each active node in the cluster has an accurate view of other node's and their status
1019
1020 Params:
1021 nodes, integer array with position of the ONOS nodes in the CLIs array
1022 """
1023 nodeResults = utilities.retry( main.Cluster.nodesCheck,
1024 False,
1025 attempts=10,
1026 sleep=10 )
1027 utilities.assert_equals( expect=True, actual=nodeResults,
1028 onpass="Nodes check successful",
1029 onfail="Nodes check NOT successful" )
1030
1031 if not nodeResults:
1032 for ctrl in main.Cluster.runningNodes:
1033 main.log.debug( "{} components not ACTIVE: \n{}".format(
1034 ctrl.name,
Jon Hall6c9e2da2018-11-06 12:01:23 -08001035 ctrl.CLI.sendline( "onos:scr-list | grep -v ACTIVE" ) ) )
You Wang0bed5932018-12-11 14:45:41 -08001036 main.log.error( "Failed to verify nodes, stopping test" )
Jon Halla604fd42018-05-04 14:27:27 -07001037 main.cleanAndExit()
1038
1039 @staticmethod
1040 def verifyTopology( main, switches, links, expNodes ):
1041 """
1042 Verifies that the ONOS cluster has an acuurate view of the topology
1043
1044 Params:
1045 switches, links, expNodes: number of expected switches, links, and nodes at this point in the test ex.: '4', '6', '2'
1046 """
1047 main.step( "Check number of topology elements" )
1048 topology = utilities.retry( main.Cluster.active( 0 ).CLI.checkStatus,
1049 main.FALSE,
1050 kwargs={ 'numoswitch': switches,
1051 'numolink': links,
1052 'numoctrl': expNodes },
1053 attempts=10,
1054 sleep=12 )
1055 utilities.assert_equals( expect=main.TRUE, actual=topology,
1056 onpass="Number of topology elements are correct",
1057 onfail="Unexpected number of links, switches, and/or controllers" )
1058
1059 @staticmethod
1060 def killOnos( main, nodes, switches, links, expNodes, sleep=None ):
Jon Hall1efcb3f2016-08-23 13:42:15 -07001061 """
1062 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
1063 switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
1064 Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
1065 """
Jon Halla604fd42018-05-04 14:27:27 -07001066 # TODO: We have enough information in the Cluster instance to remove expNodes from here and verifyTopology
Jon Hall3c910162018-03-07 14:42:16 -08001067 main.step( "Killing ONOS instances with index(es): {}".format( nodes ) )
Jon Halla604fd42018-05-04 14:27:27 -07001068 if sleep is None:
1069 sleep = float( main.params[ 'timers' ][ 'OnosDiscovery' ] )
1070 else:
1071 sleep = float( sleep )
Pier3b58c652016-09-26 12:03:31 -07001072
Jon Hall1efcb3f2016-08-23 13:42:15 -07001073 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -07001074 killResult = main.ONOSbench.onosDie( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001075 utilities.assert_equals( expect=main.TRUE, actual=killResult,
1076 onpass="ONOS instance Killed",
1077 onfail="Error killing ONOS instance" )
Devin Lim142b5342017-07-20 15:22:39 -07001078 main.Cluster.runningNodes[ i ].active = False
Jon Halla604fd42018-05-04 14:27:27 -07001079 main.Cluster.reset()
Jon Hall43060f62020-06-23 13:13:33 -07001080 main.log.debug( "sleeping %i seconds" % ( sleep ) )
Jon Halla604fd42018-05-04 14:27:27 -07001081 time.sleep( sleep )
Pier3b58c652016-09-26 12:03:31 -07001082
Devin Lim142b5342017-07-20 15:22:39 -07001083 if len( nodes ) < main.Cluster.numCtrls:
Jon Halla604fd42018-05-04 14:27:27 -07001084 Testcaselib.verifyNodes( main )
1085 Testcaselib.verifyTopology( main, switches, links, expNodes )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001086
1087 @staticmethod
Jon Halla604fd42018-05-04 14:27:27 -07001088 def recoverOnos( main, nodes, switches, links, expNodes, sleep=None ):
Jon Hall1efcb3f2016-08-23 13:42:15 -07001089 """
1090 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
1091 switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
1092 Recover an ONOS instance and verify the ONOS cluster can see the proper change
1093 """
Jon Hall3c910162018-03-07 14:42:16 -08001094 main.step( "Recovering ONOS instances with index(es): {}".format( nodes ) )
Jon Halla604fd42018-05-04 14:27:27 -07001095 if sleep is None:
1096 sleep = float( main.params[ 'timers' ][ 'OnosDiscovery' ] )
1097 else:
1098 sleep = float( sleep )
Devin Lim142b5342017-07-20 15:22:39 -07001099 [ main.ONOSbench.onosStart( main.Cluster.runningNodes[ i ].ipAddress ) for i in nodes ]
Jon Hall43060f62020-06-23 13:13:33 -07001100 main.log.debug( "sleeping %i seconds" % ( sleep ) )
Jon Halla604fd42018-05-04 14:27:27 -07001101 time.sleep( sleep )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001102 for i in nodes:
Devin Lim142b5342017-07-20 15:22:39 -07001103 isUp = main.ONOSbench.isup( main.Cluster.runningNodes[ i ].ipAddress )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001104 utilities.assert_equals( expect=main.TRUE, actual=isUp,
1105 onpass="ONOS service is ready",
1106 onfail="ONOS service did not start properly" )
1107 for i in nodes:
1108 main.step( "Checking if ONOS CLI is ready" )
Devin Lim142b5342017-07-20 15:22:39 -07001109 ctrl = main.Cluster.runningNodes[ i ]
Jonghwan Hyun76a02b72018-01-30 16:40:48 +09001110 # ctrl.CLI.startCellCli()
Devin Lim142b5342017-07-20 15:22:39 -07001111 cliResult = ctrl.CLI.startOnosCli( ctrl.ipAddress,
1112 commandlineTimeout=60,
1113 onosStartTimeout=100 )
1114 ctrl.active = True
Jon Hall1efcb3f2016-08-23 13:42:15 -07001115 utilities.assert_equals( expect=main.TRUE,
1116 actual=cliResult,
1117 onpass="ONOS CLI is ready",
1118 onfail="ONOS CLI is not ready" )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001119
Jon Halla604fd42018-05-04 14:27:27 -07001120 main.Cluster.reset()
Pier3b58c652016-09-26 12:03:31 -07001121 main.step( "Checking ONOS nodes" )
Jon Halla604fd42018-05-04 14:27:27 -07001122 Testcaselib.verifyNodes( main )
1123 Testcaselib.verifyTopology( main, switches, links, expNodes )
Pier3b58c652016-09-26 12:03:31 -07001124
Devin Lim142b5342017-07-20 15:22:39 -07001125 ready = utilities.retry( main.Cluster.active( 0 ).CLI.summary,
You Wang5bf49592020-07-08 18:47:46 -07001126 [ None, main.FALSE ],
Devin Lim142b5342017-07-20 15:22:39 -07001127 attempts=10,
1128 sleep=12 )
1129 if ready:
1130 ready = main.TRUE
1131 utilities.assert_equals( expect=main.TRUE, actual=ready,
Jon Hall1efcb3f2016-08-23 13:42:15 -07001132 onpass="ONOS summary command succeded",
1133 onfail="ONOS summary command failed" )
1134 if not ready:
1135 main.log.error( "ONOS startup failed!" )
Devin Lim44075962017-08-11 10:56:37 -07001136 main.cleanAndExit()
Jon Hall1efcb3f2016-08-23 13:42:15 -07001137
1138 @staticmethod
1139 def addHostCfg( main ):
1140 """
1141 Adds Host Configuration to ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001142 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001143 """
1144 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001145 hostCfg = {}
Devin Lim57221b02018-02-14 15:45:36 -08001146 with open( main.configPath + main.forJson + "extra.json" ) as template:
Jon Hall1efcb3f2016-08-23 13:42:15 -07001147 hostCfg = json.load( template )
1148 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'in1' ]
1149 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001150 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -07001151 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
1152 subjectClass="hosts",
1153 subjectKey=urllib.quote( mac,
1154 safe='' ),
1155 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001156 main.pingChart[ 'ip' ][ 'hosts' ] += [ 'out1' ]
1157 main.step( "Pushing new configuration" )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001158 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -07001159 main.Cluster.active( 0 ).REST.setNetCfg( cfg[ 'basic' ],
1160 subjectClass="hosts",
1161 subjectKey=urllib.quote( mac,
1162 safe='' ),
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001163 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001164 main.pingChart.update( { 'vlan1': { "expect": "True",
1165 "hosts": [ "olt1", "vsg1" ] } } )
1166 main.pingChart[ 'vlan5' ][ 'expect' ] = 0
1167 main.pingChart[ 'vlan10' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001168 ports = "[%s,%s]" % ( 5, 6 )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001169 cfg = '{"of:0000000000000001":[{"vlan":1,"ports":%s,"name":"OLT 1"}]}' % ports
Devin Lim142b5342017-07-20 15:22:39 -07001170 main.Cluster.active( 0 ).REST.setNetCfg( json.loads( cfg ),
1171 subjectClass="apps",
1172 subjectKey="org.onosproject.segmentrouting",
1173 configKey="xconnect" )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001174
1175 @staticmethod
1176 def delHostCfg( main ):
1177 """
1178 Removest Host Configuration from ONOS
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001179 Updates expected state of the network ( pingChart )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001180 """
1181 import json
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001182 hostCfg = {}
Devin Lim57221b02018-02-14 15:45:36 -08001183 with open( main.configPath + main.forJson + "extra.json" ) as template:
Jon Hall1efcb3f2016-08-23 13:42:15 -07001184 hostCfg = json.load( template )
1185 main.step( "Removing host configuration" )
1186 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001187 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -07001188 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
1189 subjectKey=urllib.quote(
1190 mac,
1191 safe='' ),
1192 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001193 main.step( "Removing configuration" )
1194 main.pingChart[ 'ip' ][ 'expect' ] = 0
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07001195 mac, cfg = hostCfg[ 'hosts' ].popitem()
Devin Lim142b5342017-07-20 15:22:39 -07001196 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="hosts",
1197 subjectKey=urllib.quote(
1198 mac,
1199 safe='' ),
1200 configKey="basic" )
Jon Hall1efcb3f2016-08-23 13:42:15 -07001201 main.step( "Removing vlan configuration" )
1202 main.pingChart[ 'vlan1' ][ 'expect' ] = 0
Devin Lim142b5342017-07-20 15:22:39 -07001203 main.Cluster.active( 0 ).REST.removeNetCfg( subjectClass="apps",
1204 subjectKey="org.onosproject.segmentrouting",
1205 configKey="xconnect" )
You Wang53dba1e2018-02-02 17:45:44 -08001206
1207 @staticmethod
1208 def verifyNetworkHostIp( main, attempts=10, sleep=10 ):
1209 """
1210 Verifies IP address assignment from the hosts
1211 """
1212 main.step( "Verify IP address assignment from hosts" )
1213 ipResult = main.TRUE
You Wangd66de192018-04-30 17:30:12 -07001214 main.Network.update()
You Wang6acb7a42018-05-04 15:12:25 -07001215 # Find out names of disconnected hosts
1216 disconnectedHosts = []
1217 if hasattr( main, "disconnectedIpv4Hosts" ):
1218 for host in main.disconnectedIpv4Hosts:
1219 disconnectedHosts.append( host )
1220 if hasattr( main, "disconnectedIpv6Hosts" ):
1221 for host in main.disconnectedIpv6Hosts:
1222 disconnectedHosts.append( host )
You Wang53dba1e2018-02-02 17:45:44 -08001223 for hostName, ip in main.expectedHosts[ "network" ].items():
You Wang6acb7a42018-05-04 15:12:25 -07001224 # Exclude disconnected hosts
1225 if hostName in disconnectedHosts:
1226 main.log.debug( "Skip verifying IP for {} as it's disconnected".format( hostName ) )
1227 continue
You Wang53dba1e2018-02-02 17:45:44 -08001228 ipResult = ipResult and utilities.retry( main.Network.verifyHostIp,
1229 main.FALSE,
1230 kwargs={ 'hostList': [ hostName ],
You Wangd66de192018-04-30 17:30:12 -07001231 'prefix': ip,
Jon Hall43060f62020-06-23 13:13:33 -07001232 'update': True },
You Wang53dba1e2018-02-02 17:45:44 -08001233 attempts=attempts,
1234 sleep=sleep )
1235 utilities.assert_equals( expect=main.TRUE, actual=ipResult,
1236 onpass="Verify network host IP succeded",
1237 onfail="Verify network host IP failed" )
1238
1239 @staticmethod
You Wangaec6a092018-08-06 15:36:31 -07001240 def verifyOnosHostIp( main, attempts=10, sleep=10, skipOnFail=True ):
You Wang53dba1e2018-02-02 17:45:44 -08001241 """
1242 Verifies host IP address assignment from ONOS
1243 """
1244 main.step( "Verify host IP address assignment in ONOS" )
1245 ipResult = main.TRUE
You Wang6acb7a42018-05-04 15:12:25 -07001246 # Find out IPs of disconnected hosts
1247 disconnectedIps = []
1248 if hasattr( main, "disconnectedIpv4Hosts" ):
1249 for host in main.disconnectedIpv4Hosts:
1250 disconnectedIps.append( main.expectedHosts[ "network" ][ host ] )
1251 if hasattr( main, "disconnectedIpv6Hosts" ):
1252 for host in main.disconnectedIpv6Hosts:
1253 disconnectedIps.append( main.expectedHosts[ "network" ][ host ] )
You Wang53dba1e2018-02-02 17:45:44 -08001254 for hostName, ip in main.expectedHosts[ "onos" ].items():
You Wang6acb7a42018-05-04 15:12:25 -07001255 # Exclude disconnected hosts
1256 if ip in disconnectedIps:
1257 main.log.debug( "Skip verifying IP for {} as it's disconnected".format( ip ) )
1258 continue
You Wang53dba1e2018-02-02 17:45:44 -08001259 ipResult = ipResult and utilities.retry( main.Cluster.active( 0 ).verifyHostIp,
1260 main.FALSE,
1261 kwargs={ 'hostList': [ hostName ],
1262 'prefix': ip },
1263 attempts=attempts,
1264 sleep=sleep )
1265 utilities.assert_equals( expect=main.TRUE, actual=ipResult,
1266 onpass="Verify ONOS host IP succeded",
1267 onfail="Verify ONOS host IP failed" )
You Wangaec6a092018-08-06 15:36:31 -07001268 if not ipResult and skipOnFail:
1269 Testcaselib.saveOnosDiagnostics( main )
You Wang89477152018-08-07 14:19:02 -07001270 Testcaselib.cleanup( main, copyKarafLog=False )
You Wangaec6a092018-08-06 15:36:31 -07001271 main.skipCase()
Andreas Pantelopoulos2eae3242018-03-06 13:47:20 -08001272
Jonghwan Hyun812c70f2018-02-16 16:33:16 -08001273 @staticmethod
1274 def updateIntfCfg( main, connectPoint, ips=[], untagged=0, tagged=[], native=0 ):
1275 """
1276 Description:
1277 Updates interface configuration in ONOS, with given IP and vlan parameters
1278 Required:
1279 * connectPoint: connect point to update configuration
1280 Optional:
1281 * ips: list of IP addresses, combined with '/xx' subnet representation,
1282 corresponding to 'ips' field in the configuration
1283 * untagged: vlan ID as an integer, corresponding to 'vlan-untagged' field in the configuration
1284 * tagged: integer list of vlan IDs, corresponding to 'vlan-tagged' field in the configuration
1285 * native: vlan ID as an integer, corresponding to 'vlan-native' field in the configuration
1286 """
1287 cfg = dict()
1288 cfg[ "ports" ] = dict()
1289 cfg[ "ports" ][ connectPoint ] = dict()
1290 cfg[ "ports" ][ connectPoint ][ "interfaces" ] = [ dict() ]
1291 cfg[ "ports" ][ connectPoint ][ "interfaces" ][ 0 ][ "ips" ] = ips
1292 if untagged > 0:
1293 cfg[ "ports" ][ connectPoint ][ "interfaces" ][ 0 ][ "vlan-untagged" ] = untagged
1294 else:
1295 cfg[ "ports" ][ connectPoint ][ "interfaces" ][ 0 ][ "vlan-tagged" ] = tagged
1296 if native > 0:
1297 cfg[ "ports" ][ connectPoint ][ "interfaces" ][ 0 ][ "vlan-native" ] = native
1298
1299 main.Cluster.active( 0 ).REST.setNetCfg( json.loads( json.dumps( cfg ) ) )
You Wange24d6272018-03-27 21:18:50 -07001300
1301 @staticmethod
You Wang2cb70172018-07-25 16:44:13 -07001302 def startScapyHosts( main, scapyNames=[], mininetNames=[] ):
You Wange24d6272018-03-27 21:18:50 -07001303 """
1304 Create host components and start Scapy CLIs
You Wang2cb70172018-07-25 16:44:13 -07001305 scapyNames: list of names that will be used as component names for scapy hosts
1306 mininetNames: used when scapy host names are different from the host names
1307 in Mininet. E.g. when scapyNames=['h1Scapy'], it's required to specify the
1308 name of the corresponding Mininet host by mininetNames=['h1']
You Wange24d6272018-03-27 21:18:50 -07001309 """
1310 main.step( "Start Scapy CLIs" )
You Wang4cc61912018-08-28 10:10:58 -07001311 main.scapyNames = scapyNames if scapyNames else main.params[ 'SCAPY' ][ 'HOSTNAMES' ].split( ',' )
1312 main.scapyHosts = [] if not hasattr( main, "scapyHosts" ) else main.scapyHosts
You Wang2cb70172018-07-25 16:44:13 -07001313 for scapyName in main.scapyNames:
You Wang4cc61912018-08-28 10:10:58 -07001314 if hasattr( main, 'Mininet1' ):
1315 main.Scapy.createHostComponent( scapyName )
1316 scapyHandle = getattr( main, scapyName )
1317 if mininetNames:
1318 mininetName = mininetNames[ scapyNames.index( scapyName ) ]
1319 else:
1320 mininetName = None
Jon Hall43060f62020-06-23 13:13:33 -07001321 if 'MN_DOCKER' in main.params and main.params['MN_DOCKER']['args']:
1322 scapyHandle.mExecDir = "/tmp"
1323 scapyHandle.hostHome = main.params[ "MN_DOCKER" ][ "home" ]
1324 main.log.debug( "start mn host component in docker" )
1325 scapyHandle.startHostCli( mininetName,
1326 execDir="/tmp",
1327 hostHome=main.params[ "MN_DOCKER" ][ "home" ] )
1328 else:
1329 main.log.debug( "start mn host component" )
1330 scapyHandle.startHostCli( mininetName )
You Wang2cb70172018-07-25 16:44:13 -07001331 else:
You Wang0fc21702018-11-02 17:49:18 -07001332 main.Network.createHostComponent( scapyName )
You Wang4cc61912018-08-28 10:10:58 -07001333 scapyHandle = getattr( main, scapyName )
1334 scapyHandle.connectInband()
1335 main.scapyHosts.append( scapyHandle )
You Wang2cb70172018-07-25 16:44:13 -07001336 scapyHandle.startScapy()
1337 scapyHandle.updateSelf()
1338 main.log.debug( scapyHandle.name )
1339 main.log.debug( scapyHandle.hostIp )
1340 main.log.debug( scapyHandle.hostMac )
1341
1342 @staticmethod
1343 def verifyTraffic( main, srcHosts, dstIp, dstHost, dstIntf, ipv6=False, expect=True, skipOnFail=True, maxRetry=2 ):
1344 """
1345 Verify unicast traffic by pinging from source hosts to the destination IP
1346 and capturing the packets at the destination host using Scapy.
1347 srcHosts: List of host names to send the ping packets
1348 dstIp: destination IP of the ping packets
1349 dstHost: host that runs Scapy to capture the packets
1350 dstIntf: name of the interface on the destination host
1351 expect: use True if the ping is expected to be captured at destination;
1352 Otherwise False
1353 skipOnFail: skip the rest of this test case if result is not expected
1354 maxRetry: number of retries allowed
1355 """
1356 from tests.dependencies.topology import Topology
1357 try:
1358 main.topo
1359 except ( NameError, AttributeError ):
1360 main.topo = Topology()
1361 main.step( "Verify traffic to {} by capturing packets on {}".format( dstIp, dstHost ) )
1362 result = main.TRUE
1363 for srcHost in srcHosts:
1364 trafficResult = main.topo.pingAndCapture( srcHost, dstIp, dstHost, dstIntf, ipv6,
1365 expect, maxRetry, True )
1366 if not trafficResult:
You Wang2cb70172018-07-25 16:44:13 -07001367 result = main.FALSE
1368 main.log.warn( "Scapy result from {} to {} is not as expected".format( srcHost, dstIp ) )
1369 utilities.assert_equals( expect=main.TRUE,
1370 actual=result,
1371 onpass="Verify traffic to {}: Pass".format( dstIp ),
1372 onfail="Verify traffic to {}: Fail".format( dstIp ) )
1373 if skipOnFail and result != main.TRUE:
1374 Testcaselib.saveOnosDiagnostics( main )
1375 Testcaselib.cleanup( main, copyKarafLog=False )
1376 main.skipCase()
You Wange24d6272018-03-27 21:18:50 -07001377
1378 @staticmethod
You Wang547893e2018-05-08 13:34:59 -07001379 def verifyMulticastTraffic( main, routeName, expect, skipOnFail=True, maxRetry=1 ):
You Wange24d6272018-03-27 21:18:50 -07001380 """
1381 Verify multicast traffic using scapy
1382 """
You Wangc564c6f2018-05-01 15:24:57 -07001383 from tests.dependencies.topology import Topology
1384 try:
1385 main.topo
1386 except ( NameError, AttributeError ):
1387 main.topo = Topology()
You Wang85747762018-05-11 15:51:50 -07001388 main.step( "Verify {} multicast traffic".format( routeName ) )
You Wangc02d8352018-04-17 16:42:10 -07001389 routeData = main.multicastConfig[ routeName ]
1390 srcs = main.mcastRoutes[ routeName ][ "src" ]
1391 dsts = main.mcastRoutes[ routeName ][ "dst" ]
1392 main.log.info( "Sending multicast traffic from {} to {}".format( [ routeData[ "src" ][ i ][ "host" ] for i in srcs ],
1393 [ routeData[ "dst" ][ i ][ "host" ] for i in dsts ] ) )
You Wang85747762018-05-11 15:51:50 -07001394 result = main.TRUE
You Wangc02d8352018-04-17 16:42:10 -07001395 for src in srcs:
1396 srcEntry = routeData[ "src" ][ src ]
1397 for dst in dsts:
1398 dstEntry = routeData[ "dst" ][ dst ]
1399 sender = getattr( main, srcEntry[ "host" ] )
1400 receiver = getattr( main, dstEntry[ "host" ] )
1401 main.Network.addRoute( str( srcEntry[ "host" ] ),
1402 str( routeData[ "group" ] ),
1403 str( srcEntry[ "interface" ] ),
1404 True if routeData[ "ipVersion" ] == 6 else False )
1405 # Build the packet
1406 sender.buildEther( dst=str( srcEntry[ "Ether" ] ) )
1407 if routeData[ "ipVersion" ] == 4:
1408 sender.buildIP( dst=str( routeData[ "group" ] ) )
1409 elif routeData[ "ipVersion" ] == 6:
1410 sender.buildIPv6( dst=str( routeData[ "group" ] ) )
1411 sender.buildUDP( ipVersion=routeData[ "ipVersion" ], dport=srcEntry[ "UDP" ] )
1412 sIface = srcEntry[ "interface" ]
1413 dIface = dstEntry[ "interface" ] if "interface" in dstEntry.keys() else None
1414 pktFilter = srcEntry[ "filter" ]
1415 pkt = srcEntry[ "packet" ]
1416 # Send packet and check received packet
1417 expectedResult = expect.pop( 0 ) if isinstance( expect, list ) else expect
You Wangc564c6f2018-05-01 15:24:57 -07001418 t3Cmd = "t3-troubleshoot -vv -sp {} -et ipv{} -d {} -dm {}".format( srcEntry[ "port" ], routeData[ "ipVersion" ],
Jon Halla604fd42018-05-04 14:27:27 -07001419 routeData[ "group" ], srcEntry[ "Ether" ] )
You Wangc564c6f2018-05-01 15:24:57 -07001420 trafficResult = main.topo.sendScapyPackets( sender, receiver, pktFilter, pkt, sIface, dIface,
1421 expectedResult, maxRetry, True, t3Cmd )
You Wang85747762018-05-11 15:51:50 -07001422 if not trafficResult:
1423 result = main.FALSE
1424 main.log.warn( "Scapy result from {} to {} is not as expected".format( srcEntry[ "host" ],
1425 dstEntry[ "host" ] ) )
1426 utilities.assert_equals( expect=main.TRUE,
1427 actual=result,
1428 onpass="Verify {} multicast traffic: Pass".format( routeName ),
1429 onfail="Verify {} multicast traffic: Fail".format( routeName ) )
You Wangba823f02018-05-17 13:54:08 -07001430 if skipOnFail and result != main.TRUE:
You Wang85747762018-05-11 15:51:50 -07001431 Testcaselib.saveOnosDiagnostics( main )
1432 Testcaselib.cleanup( main, copyKarafLog=False )
1433 main.skipCase()
You Wangc02d8352018-04-17 16:42:10 -07001434
1435 @staticmethod
You Wang85747762018-05-11 15:51:50 -07001436 def verifyPing( main, srcList, dstList, ipv6=False, expect=True, wait=1,
You Wang54b1d672018-06-11 16:44:13 -07001437 acceptableFailed=0, skipOnFail=True, stepMsg="Verify Ping",
1438 t3Simple=True ):
You Wang5da39c82018-04-26 22:55:08 -07001439 """
1440 Verify reachability from each host in srcList to each host in dstList
1441 """
1442 from tests.dependencies.topology import Topology
1443 try:
1444 main.topo
1445 except ( NameError, AttributeError ):
1446 main.topo = Topology()
You Wang85747762018-05-11 15:51:50 -07001447 main.step( stepMsg )
You Wang54b1d672018-06-11 16:44:13 -07001448 pingResult = main.topo.ping( srcList, dstList, ipv6, expect, wait, acceptableFailed, skipOnFail, t3Simple )
You Wang85747762018-05-11 15:51:50 -07001449 utilities.assert_equals( expect=main.TRUE,
1450 actual=pingResult,
1451 onpass="{}: Pass".format( stepMsg ),
1452 onfail="{}: Fail".format( stepMsg ) )
You Wang5da39c82018-04-26 22:55:08 -07001453 if not pingResult and skipOnFail:
1454 Testcaselib.saveOnosDiagnostics( main )
1455 Testcaselib.cleanup( main, copyKarafLog=False, removeHostComponent=True )
1456 main.skipCase()
You Wangbc898b82018-05-03 16:22:34 -07001457
1458 @staticmethod
You Wang85747762018-05-11 15:51:50 -07001459 def verifyHostLocations( main, locationDict, retry=2 ):
You Wangbc898b82018-05-03 16:22:34 -07001460 """
1461 Verify if the specified host is discovered by ONOS on the given locations
1462 Required:
You Wang85747762018-05-11 15:51:50 -07001463 locationDict: a dictionary that maps host names to expected locations.
1464 locations could be a string or a list.
1465 ex. { "h1v4": ["of:0000000000000005/8"] }
You Wangbc898b82018-05-03 16:22:34 -07001466 Returns:
1467 main.TRUE if host is discovered on all locations provided, otherwise main.FALSE
1468 """
You Wang85747762018-05-11 15:51:50 -07001469 main.step( "Verify locations of hosts {}".format( locationDict.keys() ) )
1470 result = main.TRUE
1471 for hostName, locations in locationDict.items():
1472 main.log.info( "Verify host {} is discovered at {}".format( hostName, locations ) )
1473 hostIp = main.Network.getIPAddress( hostName, proto='IPV4' )
1474 if not hostIp:
1475 hostIp = main.Network.getIPAddress( hostName, proto='IPV6' )
1476 if not hostIp:
1477 main.log.warn( "Failed to find IP address for host {}, skipping location verification".format( hostName ) )
1478 result = main.FALSE
1479 continue
1480 locationResult = utilities.retry( main.Cluster.active( 0 ).CLI.verifyHostLocation,
1481 main.FALSE,
1482 args=( hostIp, locations ),
1483 attempts=retry + 1,
1484 sleep=10 )
1485 if not locationResult:
1486 result = main.FALSE
1487 main.log.warn( "location verification for host {} failed".format( hostName ) )
You Wang547893e2018-05-08 13:34:59 -07001488 utilities.assert_equals( expect=main.TRUE, actual=result,
You Wang85747762018-05-11 15:51:50 -07001489 onpass="Location verification passed",
1490 onfail="Location verification failed" )
Jonghwan Hyun785471d2018-05-14 14:48:19 -07001491
1492 @staticmethod
You Wang6e5b48e2018-07-23 16:17:38 -07001493 def moveHost( main, hostName, srcSw, dstSw, gw, macAddr=None, prefixLen=None, cfg='', ipv6=False, vlan=None ):
Jonghwan Hyun785471d2018-05-14 14:48:19 -07001494 """
1495 Move specified host from srcSw to dstSw.
1496 If srcSw and dstSw are same, the host will be moved from current port to
1497 next available port.
1498 Required:
1499 hostName: name of the host. e.g., "h1"
1500 srcSw: name of the switch that the host is attached to. e.g., "leaf1"
1501 dstSw: name of the switch that the host will be moved to. e.g., "leaf2"
1502 gw: ip address of the gateway of the new location
1503 Optional:
1504 macAddr: if specified, change MAC address of the host to the specified MAC address.
1505 prefixLen: prefix length
1506 cfg: port configuration as JSON string
1507 ipv6: Use True to move IPv6 host
You Wang6e5b48e2018-07-23 16:17:38 -07001508 vlan: vlan number of the host
Jonghwan Hyun785471d2018-05-14 14:48:19 -07001509 """
Jonghwan Hyun785471d2018-05-14 14:48:19 -07001510 if not hasattr( main, 'Mininet1' ):
1511 main.log.warn( "moveHost is supposed to be used only in Mininet." )
1512 return
1513
You Wang6e5b48e2018-07-23 16:17:38 -07001514 main.step( "Moving {} host {} from {} to {}".format( 'tagged' if vlan else 'untagged', hostName, srcSw, dstSw ) )
1515 main.Mininet1.moveHost( hostName, srcSw, dstSw, macAddr, prefixLen, ipv6, vlan=vlan )
1516 if not ipv6:
You Wang6260ed52018-07-18 17:54:25 -07001517 main.Mininet1.changeDefaultGateway( hostName, gw )
Jonghwan Hyun785471d2018-05-14 14:48:19 -07001518 if cfg:
1519 main.Cluster.active( 0 ).REST.setNetCfg( json.loads( cfg ),
1520 subjectClass="ports" )
You Wang6260ed52018-07-18 17:54:25 -07001521 # Wait for the host to get RA for setting up default gateway
Jon Hall43060f62020-06-23 13:13:33 -07001522 main.log.debug( "sleeping %i seconds" % ( 5 ) )
You Wang6260ed52018-07-18 17:54:25 -07001523 time.sleep( 5 )
Jonghwan Hyun785471d2018-05-14 14:48:19 -07001524
1525 main.Mininet1.discoverHosts( [ hostName, ] )
1526
1527 # Update expectedHost when MAC address is changed.
1528 if macAddr is not None:
1529 ipAddr = main.expectedHosts[ "network" ][ hostName ]
1530 if ipAddr is not None:
1531 for hostName, ip in main.expectedHosts[ "onos" ].items():
1532 if ip == ipAddr:
1533 vlan = hostName.split( "/" )[ -1 ]
1534 del main.expectedHosts[ "onos" ][ hostName ]
You Wang7ea90582018-07-19 15:27:58 -07001535 main.expectedHosts[ "onos" ][ "{}/{}".format( macAddr.upper(), vlan ) ] = ip
Jonghwan Hyun785471d2018-05-14 14:48:19 -07001536 break
Jonghwan Hyun5fad1782018-05-21 14:11:16 -07001537
1538 @staticmethod
1539 def moveDualHomedHost( main, hostName, srcSw, srcPairSw, dstSw, dstPairSw, gw,
You Wang6e5b48e2018-07-23 16:17:38 -07001540 macAddr=None, prefixLen=24, cfg='', ipv6=False, vlan=None ):
Jonghwan Hyun5fad1782018-05-21 14:11:16 -07001541 """
1542 Move specified dual-homed host from srcSw-srcPairSw to dstSw-dstPairSw.
1543 If srcSw-srcPairSw and dstSw-dstPairSw are same, the host will be moved from current port
1544 to next available port.
1545 Required:
1546 hostName: name of the host. e.g., "h1"
1547 srcSw: name of the switch that the host is attached to. e.g., "leaf1"
1548 srcPairSw: name of the paired-switch that the host is attached to. e.g., "leaf2"
1549 dstSw: name of the switch that the host will be moved to. e.g., "leaf1"
1550 dstPairSw: name of the paired-switch that the host will be moved to. e.g., "leaf2"
1551 gw: ip address of the gateway of the new location
1552 Optional:
1553 macAddr: if specified, change MAC address of the host to the specified MAC address.
1554 prefixLen: prefix length
1555 cfg: port configurations as JSON string
You Wang7ea90582018-07-19 15:27:58 -07001556 ipv6: Use True to move IPv6 host
You Wang6e5b48e2018-07-23 16:17:38 -07001557 vlan: vlan number of the host
Jonghwan Hyun5fad1782018-05-21 14:11:16 -07001558 """
Jonghwan Hyun5fad1782018-05-21 14:11:16 -07001559 if not hasattr( main, 'Mininet1' ):
1560 main.log.warn( "moveDualHomedHost is supposed to be used only in Mininet." )
1561 return
1562
You Wang6e5b48e2018-07-23 16:17:38 -07001563 main.step( "Moving {} host {} from {} and {} to {} and {}".format( 'tagged' if vlan else 'untagged', hostName,
1564 srcSw, srcPairSw, dstSw, dstPairSw ) )
1565 main.Mininet1.moveDualHomedHost( hostName, srcSw, srcPairSw, dstSw, dstPairSw,
1566 macAddr=macAddr, prefixLen=prefixLen, ipv6=ipv6, vlan=vlan )
1567 if not ipv6:
You Wang7ea90582018-07-19 15:27:58 -07001568 main.Mininet1.changeDefaultGateway( hostName, gw )
Jonghwan Hyun5fad1782018-05-21 14:11:16 -07001569 if cfg:
1570 main.Cluster.active( 0 ).REST.setNetCfg( json.loads( cfg ),
1571 subjectClass="ports" )
You Wang7ea90582018-07-19 15:27:58 -07001572 # Wait for the host to get RA for setting up default gateway
Jon Hall43060f62020-06-23 13:13:33 -07001573 main.log.debug( "sleeping %i seconds" % ( 5 ) )
You Wang7ea90582018-07-19 15:27:58 -07001574 time.sleep( 5 )
Jonghwan Hyun5fad1782018-05-21 14:11:16 -07001575
1576 main.Mininet1.discoverHosts( [ hostName, ] )
1577
1578 # Update expectedHost when MAC address is changed.
1579 if macAddr is not None:
1580 ipAddr = main.expectedHosts[ "network" ][ hostName ]
1581 if ipAddr is not None:
1582 for hostName, ip in main.expectedHosts[ "onos" ].items():
1583 if ip == ipAddr:
1584 vlan = hostName.split( "/" )[ -1 ]
1585 del main.expectedHosts[ "onos" ][ hostName ]
You Wang7ea90582018-07-19 15:27:58 -07001586 main.expectedHosts[ "onos" ][ "{}/{}".format( macAddr.upper(), vlan ) ] = ip
Jon Hall43060f62020-06-23 13:13:33 -07001587
1588 @staticmethod
1589 def mnDockerSetup( main ):
1590 """
1591 Optionally start and setup docker image for mininet
1592 """
1593 if 'MN_DOCKER' in main.params and main.params['MN_DOCKER']['args']:
1594
1595 main.log.info( "Creating Mininet Docker" )
1596 handle = main.Mininet1.handle
1597 main.Mininet1.dockerPrompt = '#'
Jon Hallf6aeda22020-07-28 09:12:56 -07001598 # build docker image
1599 buildOutput = ""
1600 try:
1601 handle.sendline( " docker build -t trellis_mininet %s/../dependencies/" % main.testDir )
1602 handle.expect( "Successfully built", timeout=600 )
1603 buildOutput = handle.before + str( handle.after )
1604 handle.expect( main.Mininet1.prompt )
1605 buildOutput += handle.before
1606 main.log.debug( buildOutput )
1607 except pexpect.TIMEOUT as e:
1608 main.log.error( e )
1609 buildOutput += handle.before
1610 main.log.debug( buildOutput )
1611
Jon Hall43060f62020-06-23 13:13:33 -07001612
1613 confDir = "/tmp/mn_conf/"
1614 # Try to ensure the destination exists
1615 main.log.info( "Create folder for network config files" )
1616 handle.sendline( "mkdir -p %s" % confDir )
1617 handle.expect( main.Mininet1.prompt )
1618 main.log.debug( handle.before + handle.after )
1619 # Make sure permissions are correct
1620 handle.sendline( "sudo chown %s:%s %s" % ( main.Mininet1.user_name, main.Mininet1.user_name, confDir ) )
Jon Hallf6aeda22020-07-28 09:12:56 -07001621 handle.expect( main.Mininet1.prompt )
Jon Hall43060f62020-06-23 13:13:33 -07001622 handle.sendline( "sudo chmod -R a+rwx %s" % ( confDir ) )
1623 handle.expect( main.Mininet1.prompt )
1624 main.log.debug( handle.before + handle.after )
Jon Hallcc166702020-07-30 11:42:22 -07001625 # Stop any leftover container
1626 handle.sendline( "docker stop trellis_mininet" )
1627 handle.expect( main.Mininet1.bashPrompt )
1628 main.log.debug( handle.before )
Jon Hall43060f62020-06-23 13:13:33 -07001629 # Start docker container
1630 handle.sendline( "docker run --name trellis_mininet %s %s" % ( main.params[ 'MN_DOCKER' ][ 'args' ], main.params[ 'MN_DOCKER' ][ 'name' ] ) )
Jon Hallf6aeda22020-07-28 09:12:56 -07001631 i = handle.expect( [ main.Mininet1.bashPrompt, "Error response from daemon: Conflict. The container name" ] )
Jon Hall43060f62020-06-23 13:13:33 -07001632 output = handle.before + handle.after
1633 main.log.debug( repr(output) )
Jon Hallf6aeda22020-07-28 09:12:56 -07001634 if i == 1:
1635 main.log.error( "Docker container already running, aborting test" )
1636 main.cleanup()
1637 main.exit()
Jon Hall43060f62020-06-23 13:13:33 -07001638
1639 handle.sendline( "docker attach trellis_mininet" )
1640 handle.expect( main.Mininet1.dockerPrompt )
1641 main.log.debug( handle.before + handle.after )
1642 handle.sendline( "sysctl -w net.ipv4.ip_forward=0" )
1643 handle.sendline( "sysctl -w net.ipv4.conf.all.forwarding=0" )
1644 handle.expect( main.Mininet1.dockerPrompt )
1645 main.log.debug( handle.before + handle.after )
1646 # We should be good to go
1647 main.Mininet1.prompt = main.Mininet1.dockerPrompt
1648 main.Mininet1.sudoRequired = False
1649
1650 # Fow when we create component handles
1651 main.Mininet1.mExecDir = "/tmp"
1652 main.Mininet1.hostHome = main.params[ "MN_DOCKER" ][ "home" ]
1653 main.Mininet1.hostPrompt = "/home/root#"
1654
1655 @staticmethod
1656 def mnDockerTeardown( main ):
1657 """
1658 Optionally stop and cleanup docker image for mininet
1659 """
1660
1661 if hasattr( main, 'Mininet1' ):
1662 if 'MN_DOCKER' in main.params and main.params['MN_DOCKER']['args']:
1663 main.log.info( "Deleting Mininet Docker" )
1664
1665 # Detach from container
1666 handle = main.Mininet1.handle
1667 try:
1668 handle.sendline( "exit" ) # ctrl-p ctrk-q to detach from container
1669 main.log.debug( "sleeping %i seconds" % ( 5 ) )
1670 time.sleep(5)
1671 handle.expect( main.Mininet1.dockerPrompt )
1672 main.log.debug( handle.before + handle.after )
1673 main.Mininet1.prompt = main.Mininet1.bashPrompt
1674 handle.expect( main.Mininet1.prompt )
1675 main.log.debug( handle.before + handle.after )
1676 main.Mininet1.sudoRequired = True
1677 except Exception as e:
1678 main.log.error( e )
1679
1680 @staticmethod
1681 def setOnosConfig( main ):
1682 """
1683 Read and Set onos configurations from the params file
1684 """
1685 main.step( "Set ONOS configurations" )
1686 config = main.params.get( 'ONOS_Configuration' )
1687 if config:
1688 main.log.debug( config )
1689 checkResult = main.TRUE
1690 for component in config:
1691 for setting in config[ component ]:
1692 value = config[ component ][ setting ]
1693 check = main.Cluster.next().setCfg( component, setting, value )
1694 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
1695 checkResult = check and checkResult
1696 utilities.assert_equals( expect=main.TRUE,
1697 actual=checkResult,
1698 onpass="Successfully set config",
1699 onfail="Failed to set config" )
1700 else:
1701 main.log.warn( "No configurations were specified to be changed after startup" )
1702
1703 @staticmethod
1704 def setOnosLogLevels( main ):
1705 """
1706 Read and Set onos log levels from the params file
1707 """
1708 main.step( 'Set logging levels' )
1709 logging = True
1710 try:
1711 logs = main.params.get( 'ONOS_Logging', False )
1712 if logs:
1713 for namespace, level in logs.items():
1714 for ctrl in main.Cluster.active():
1715 ctrl.CLI.logSet( level, namespace )
1716 except AttributeError:
1717 logging = False
1718 utilities.assert_equals( expect=True, actual=logging,
1719 onpass="Set log levels",
1720 onfail="Failed to set log levels" )