blob: 8565930efced80d45dcdf3207cf6ef3806132b40 [file] [log] [blame]
Flavio Castro0a05b8d2016-06-16 17:26:51 -07001import os
2import imp
3import time
Flavio Castro49c218d2016-06-21 16:55:57 -07004import json
Flavio Castro0a05b8d2016-06-16 17:26:51 -07005
6from core import utilities
7
8
9class Testcaselib:
10 @staticmethod
11 def initTest( main ):
12 """
13 - Construct tests variables
14 - GIT ( optional )
15 - Checkout ONOS master branch
16 - Pull latest ONOS code
17 - Building ONOS ( optional )
18 - Install ONOS package
19 - Build ONOS package
20 """
Flavio Castro0a05b8d2016-06-16 17:26:51 -070021 main.step( "Constructing test variables" )
22 # Test variables
23 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
24 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
25 main.diff = main.params[ 'ENV' ][ 'diffApps' ]
26 gitBranch = main.params[ 'GIT' ][ 'branch' ]
27 main.path = os.path.dirname( main.testFile )
Flavio Castro5608a392016-06-22 17:02:35 -070028 main.dependencyPath = main.path + "/../dependencies/"
Flavio Castro0a05b8d2016-06-16 17:26:51 -070029 main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
30 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
Flavio Castro5608a392016-06-22 17:02:35 -070031 main.scale = (main.params[ 'SCALE' ][ 'size' ]).split( "," )
Flavio Castro0a05b8d2016-06-16 17:26:51 -070032 main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
Flavio Castro5608a392016-06-22 17:02:35 -070033 # main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
Flavio Castro0a05b8d2016-06-16 17:26:51 -070034 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
Flavio Castro5608a392016-06-22 17:02:35 -070035 main.cellData = { } # for creating cell file
36 main.CLIs = [ ]
37 main.ONOSip = [ ]
38 main.RESTs = [ ]
Flavio Castro0a05b8d2016-06-16 17:26:51 -070039
40 # Assigning ONOS cli handles to a list
Flavio Castro5608a392016-06-22 17:02:35 -070041 for i in range( 1, main.maxNodes + 1 ):
Flavio Castro0a05b8d2016-06-16 17:26:51 -070042 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
Flavio Castro49c218d2016-06-21 16:55:57 -070043 main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) )
Flavio Castro5608a392016-06-22 17:02:35 -070044 main.ONOSip.append( main.CLIs[ i - 1 ].ip_address )
Flavio Castro0a05b8d2016-06-16 17:26:51 -070045 # -- INIT SECTION, ONLY RUNS ONCE -- #
46 main.startUp = imp.load_source( wrapperFile1,
47 main.dependencyPath +
48 wrapperFile1 +
49 ".py" )
50
51 copyResult1 = main.ONOSbench.scp( main.Mininet1,
52 main.dependencyPath +
53 main.topology,
54 main.Mininet1.home,
55 direction="to" )
56 if main.CLIs:
57 stepResult = main.TRUE
58 else:
59 main.log.error( "Did not properly created list of ONOS CLI handle" )
60 stepResult = main.FALSE
61
62 utilities.assert_equals( expect=main.TRUE,
63 actual=stepResult,
64 onpass="Successfully construct " +
65 "test variables ",
66 onfail="Failed to construct test variables" )
67
68 @staticmethod
69 def installOnos( main ):
70 """
71 - Set up cell
72 - Create cell file
73 - Set cell file
74 - Verify cell file
75 - Kill ONOS process
76 - Uninstall ONOS cluster
77 - Verify ONOS start up
78 - Install ONOS cluster
79 - Connect to cli
80 """
81 # main.scale[ 0 ] determines the current number of ONOS controller
Flavio Castro5608a392016-06-22 17:02:35 -070082 apps = main.apps
Flavio Castro0a05b8d2016-06-16 17:26:51 -070083 if main.diff:
84 apps = main.apps + "," + main.diff
Flavio Castro5608a392016-06-22 17:02:35 -070085 else:
86 main.log.error( "App list is empty" )
87 main.case( "Package and start ONOS using apps:" + apps )
Flavio Castro0a05b8d2016-06-16 17:26:51 -070088 print "NODE COUNT = ", main.numCtrls
89 print main.ONOSip
Flavio Castro5608a392016-06-22 17:02:35 -070090 tempOnosIp = [ ]
Flavio Castro0a05b8d2016-06-16 17:26:51 -070091 for i in range( main.numCtrls ):
Flavio Castro5608a392016-06-22 17:02:35 -070092 tempOnosIp.append( main.ONOSip[ i ] )
Flavio Castro0a05b8d2016-06-16 17:26:51 -070093 onosUser = main.params[ 'ENV' ][ 'cellUser' ]
Flavio Castro5608a392016-06-22 17:02:35 -070094 main.step( "Create and Apply cell file" )
Flavio Castro0a05b8d2016-06-16 17:26:51 -070095 main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
96 "temp",
97 main.Mininet1.ip_address,
98 apps,
99 tempOnosIp,
100 onosUser )
101 cellResult = main.ONOSbench.setCell( "temp" )
Flavio Castro5608a392016-06-22 17:02:35 -0700102 verifyResult = main.ONOSbench.verifyCell( )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700103 stepResult = cellResult and verifyResult
104 utilities.assert_equals( expect=main.TRUE,
105 actual=stepResult,
106 onpass="Successfully applied cell to " + \
107 "environment",
108 onfail="Failed to apply cell to environment " )
Flavio Castro5608a392016-06-22 17:02:35 -0700109 # kill off all onos processes
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700110 main.log.info( "Safety check, killing all ONOS processes" +
111 " before initiating environment setup" )
112 for i in range( main.maxNodes ):
113 main.ONOSbench.onosDie( main.ONOSip[ i ] )
114 main.step( "Create and Install ONOS package" )
Flavio Castro5608a392016-06-22 17:02:35 -0700115 packageResult = main.ONOSbench.onosPackage( )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700116
117 onosInstallResult = main.TRUE
118 for i in range( main.numCtrls ):
119 onosInstallResult = onosInstallResult and \
Flavio Castro5608a392016-06-22 17:02:35 -0700120 main.ONOSbench.onosInstall(
Flavio Castro519072e2016-06-23 13:30:48 -0700121 node=main.ONOSip[ i ] )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700122 stepResult = onosInstallResult
123 utilities.assert_equals( expect=main.TRUE,
124 actual=stepResult,
125 onpass="Successfully installed ONOS package",
126 onfail="Failed to install ONOS package" )
127 main.step( "Starting ONOS service" )
Flavio Castro5608a392016-06-22 17:02:35 -0700128 stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE,
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700129 for i in range( main.numCtrls ):
130 onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
131 if onosIsUp == main.TRUE:
132 main.log.report( "ONOS instance is up and ready" )
133 else:
134 main.log.report( "ONOS instance may not be up, stop and " +
135 "start ONOS again " )
136 for i in range( main.numCtrls ):
137 stopResult = stopResult and \
138 main.ONOSbench.onosStop( main.ONOSip[ i ] )
139 for i in range( main.numCtrls ):
140 startResult = startResult and \
141 main.ONOSbench.onosStart( main.ONOSip[ i ] )
142 stepResult = onosIsUp and stopResult and startResult
143
144 utilities.assert_equals( expect=main.TRUE,
145 actual=stepResult,
146 onpass="ONOS service is ready",
147 onfail="ONOS service did not start properly" )
148 main.step( "Checking if ONOS CLI is ready" )
149 for i in range( main.numCtrls ):
Flavio Castro5608a392016-06-22 17:02:35 -0700150 main.CLIs[ i ].startCellCli( )
151 cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
152 commandlineTimeout=60,
153 onosStartTimeout=100 )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700154 utilities.assert_equals( expect=main.TRUE,
155 actual=cliResult,
156 onpass="ONOS CLI is ready",
157 onfail="ONOS CLI is not ready" )
Flavio Castro5608a392016-06-22 17:02:35 -0700158 main.active = 0
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700159 for i in range( 10 ):
160 ready = True
Flavio Castro5608a392016-06-22 17:02:35 -0700161 output = main.CLIs[ main.active ].summary( )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700162 if not output:
163 ready = False
164 if ready:
165 break
166 time.sleep( 10 )
167 utilities.assert_equals( expect=True, actual=ready,
168 onpass="ONOS summary command succeded",
169 onfail="ONOS summary command failed" )
170
Flavio Castro49c218d2016-06-21 16:55:57 -0700171 with open( main.dependencyPath + "/" + main.cfgName + ".json" ) as cfg:
Flavio Castro5608a392016-06-22 17:02:35 -0700172 main.RESTs[ main.active ].setNetCfg( json.load( cfg ) )
Flavio Castro49c218d2016-06-21 16:55:57 -0700173
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700174 if not ready:
175 main.log.error( "ONOS startup failed!" )
Flavio Castro5608a392016-06-22 17:02:35 -0700176 main.cleanup( )
177 main.exit( )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700178
179 for i in range( main.numCtrls ):
Flavio Castro5608a392016-06-22 17:02:35 -0700180 main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.segmentrouting" )
181 main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.driver.pipeline" )
182 main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.store.group.impl" )
183 main.CLIs[ i ].logSet( "DEBUG",
184 "org.onosproject.net.flowobjective.impl" )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700185
186 @staticmethod
187 def startMininet( main, topology, args="" ):
188 main.step( "Starting Mininet Topology" )
189 arg = "--onos %d %s" % (main.numCtrls, args)
Flavio Castro5608a392016-06-22 17:02:35 -0700190 main.topology = topology
191 topoResult = main.Mininet1.startNet(
Flavio Castro519072e2016-06-23 13:30:48 -0700192 topoFile=main.dependencyPath + main.topology, args=arg )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700193 stepResult = topoResult
194 utilities.assert_equals( expect=main.TRUE,
195 actual=stepResult,
196 onpass="Successfully loaded topology",
197 onfail="Failed to load topology" )
198 # Exit if topology did not load properly
199 if not topoResult:
Flavio Castro5608a392016-06-22 17:02:35 -0700200 main.cleanup( )
201 main.exit( )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700202
203 @staticmethod
Flavio Castro5608a392016-06-22 17:02:35 -0700204 def checkFlows( main, minFlowCount ):
205 main.step(
Flavio Castro519072e2016-06-23 13:30:48 -0700206 " Check whether the flow count is bigger than %s" % minFlowCount )
Flavio Castro5608a392016-06-22 17:02:35 -0700207 count = utilities.retry( main.CLIs[ main.active ].checkFlowCount,
Flavio Castro02b43632016-06-20 17:07:27 -0700208 main.FALSE,
Flavio Castro5608a392016-06-22 17:02:35 -0700209 kwargs={ 'min': minFlowCount },
Flavio Castro02b43632016-06-20 17:07:27 -0700210 attempts=10,
Flavio Castro5608a392016-06-22 17:02:35 -0700211 sleep=10 )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700212 utilities.assertEquals( \
213 expect=True,
Flavio Castro5608a392016-06-22 17:02:35 -0700214 actual=(count > 0),
215 onpass="Flow count looks correct: " + str( count ),
216 onfail="Flow count looks wrong: " + str( count ) )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700217
218 main.step( "Check whether all flow status are ADDED" )
Flavio Castro5608a392016-06-22 17:02:35 -0700219 flowCheck = utilities.retry( main.CLIs[ main.active ].checkFlowsState,
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700220 main.FALSE,
Flavio Castro5608a392016-06-22 17:02:35 -0700221 kwargs={ 'isPENDING': False },
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700222 attempts=10,
Flavio Castro5608a392016-06-22 17:02:35 -0700223 sleep=10 )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700224 utilities.assertEquals( \
225 expect=main.TRUE,
226 actual=flowCheck,
227 onpass="Flow status is correct!",
228 onfail="Flow status is wrong!" )
Flavio Castro5608a392016-06-22 17:02:35 -0700229 main.ONOSbench.dumpFlows( main.ONOSip[ main.active ],
230 main.logdir, "flowsBefore" + main.cfgName )
231 main.ONOSbench.dumpGroups( main.ONOSip[ 0 ],
232 main.logdir, "groupsBefore" + main.cfgName )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700233
234 @staticmethod
Flavio Castro519072e2016-06-23 13:30:48 -0700235 def pingAll( main, tag="", dumpflows=True ):
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700236 main.log.report( "Check full connectivity" )
Flavio Castro5608a392016-06-22 17:02:35 -0700237 main.step( "Check full connectivity %s" % tag )
238 pa = main.Mininet1.pingall( )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700239 utilities.assert_equals( expect=main.TRUE, actual=pa,
240 onpass="Full connectivity successfully tested",
241 onfail="Full connectivity failed" )
Flavio Castro519072e2016-06-23 13:30:48 -0700242 if dumpflows:
243 main.ONOSbench.dumpFlows( main.ONOSip[ main.active ],
244 main.logdir, "flowsOn" + tag )
245 main.ONOSbench.dumpGroups( main.ONOSip[ main.active ],
246 main.logdir, "groupsOn" + tag )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700247
248 @staticmethod
249 def killLink( main, end1, end2, switches, links ):
250 """
251 end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
252 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
253 Kill a link and verify ONOS can see the proper link change
254 """
255 main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Flavio Castro5608a392016-06-22 17:02:35 -0700256 main.step( "Kill link between %s and %s" % (end1, end2) )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700257 LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
Flavio Castro5608a392016-06-22 17:02:35 -0700258 main.log.info(
Flavio Castro519072e2016-06-23 13:30:48 -0700259 "Waiting %s seconds for link down to be discovered" % main.linkSleep )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700260 time.sleep( main.linkSleep )
Flavio Castro5608a392016-06-22 17:02:35 -0700261 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
262 main.FALSE,
263 kwargs={ 'numoswitch': switches,
264 'numolink': links },
265 attempts=10,
266 sleep=main.linkSleep )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700267 result = topology & LinkDown
268 utilities.assert_equals( expect=main.TRUE, actual=result,
269 onpass="Link down successful",
270 onfail="Failed to turn off link?" )
271
272 @staticmethod
Flavio Castro5608a392016-06-22 17:02:35 -0700273 def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
274 links ):
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700275 """
276 Params:
277 end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
278 dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
Flavio Castro02b43632016-06-20 17:07:27 -0700279 port1, port2: respective port of the end switches that connects to the link, ex.:'1'
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700280 switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
281 Kill a link and verify ONOS can see the proper link change
282 """
Flavio Castro5608a392016-06-22 17:02:35 -0700283 main.step( "Restore link between %s and %s" % (end1, end2) )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700284 result = False
Flavio Castro5608a392016-06-22 17:02:35 -0700285 count = 0
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700286 while True:
Flavio Castro5608a392016-06-22 17:02:35 -0700287 count += 0
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700288 main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
289 main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
Flavio Castro5608a392016-06-22 17:02:35 -0700290 main.log.info(
Flavio Castro519072e2016-06-23 13:30:48 -0700291 "Waiting %s seconds for link up to be discovered" % main.linkSleep )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700292 time.sleep( main.linkSleep )
Flavio Castro5608a392016-06-22 17:02:35 -0700293 main.CLIs[ main.active ].portstate( dpid=dpid1, port=port1 )
294 main.CLIs[ main.active ].portstate( dpid=dpid2, port=port2 )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700295 time.sleep( main.linkSleep )
296
Flavio Castro5608a392016-06-22 17:02:35 -0700297 result = main.CLIs[ main.active ].checkStatus( numoswitch=switches,
298 numolink=links )
299 if count > 5 or result:
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700300 break
301 utilities.assert_equals( expect=main.TRUE, actual=result,
302 onpass="Link up successful",
303 onfail="Failed to bring link up" )
304
305 @staticmethod
306 def killSwitch( main, switch, switches, links ):
307 """
308 Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
Flavio Castro02b43632016-06-20 17:07:27 -0700309 Completely kill a switch and verify ONOS can see the proper change
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700310 """
311 main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
312 main.step( "Kill " + switch )
313 main.log.info( "Stopping" + switch )
314 main.Mininet1.switch( SW=switch, OPTION="stop" )
Flavio Castro5608a392016-06-22 17:02:35 -0700315 # todo make this repeatable
316 main.log.info( "Waiting %s seconds for switch down to be discovered" % (
Flavio Castro519072e2016-06-23 13:30:48 -0700317 main.switchSleep) )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700318 time.sleep( main.switchSleep )
Flavio Castro5608a392016-06-22 17:02:35 -0700319 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
320 main.FALSE,
321 kwargs={ 'numoswitch': switches,
322 'numolink': links },
323 attempts=10,
324 sleep=main.switchSleep )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700325 utilities.assert_equals( expect=main.TRUE, actual=topology,
326 onpass="Kill switch successful",
327 onfail="Failed to kill switch?" )
328
329 @staticmethod
330 def recoverSwitch( main, switch, switches, links ):
331 """
332 Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
333 Recover a switch and verify ONOS can see the proper change
334 """
Flavio Castro5608a392016-06-22 17:02:35 -0700335 # todo make this repeatable
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700336 main.step( "Recovering " + switch )
337 main.log.info( "Starting" + switch )
Flavio Castro5608a392016-06-22 17:02:35 -0700338 main.Mininet1.switch( SW=switch, OPTION="start" )
339 main.log.info( "Waiting %s seconds for switch up to be discovered" % (
Flavio Castro519072e2016-06-23 13:30:48 -0700340 main.switchSleep) )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700341 time.sleep( main.switchSleep )
Flavio Castro5608a392016-06-22 17:02:35 -0700342 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
343 main.FALSE,
344 kwargs={ 'numoswitch': switches,
345 'numolink': links },
346 attempts=10,
347 sleep=main.switchSleep )
Flavio Castro0a05b8d2016-06-16 17:26:51 -0700348 utilities.assert_equals( expect=main.TRUE, actual=topology,
349 onpass="Switch recovery successful",
350 onfail="Failed to recover switch?" )
351
352 @staticmethod
353 def cleanup( main ):
354 """
355 Stop Onos-cluster.
356 Stops Mininet
357 Copies ONOS log
358 """
Flavio Castro5608a392016-06-22 17:02:35 -0700359 main.Mininet1.stopNet( )
Flavio Castro49c218d2016-06-21 16:55:57 -0700360 main.ONOSbench.scp( main.ONOScli1, "/opt/onos/log/karaf.log",
Flavio Castro5608a392016-06-22 17:02:35 -0700361 "/tmp/karaf.log", direction="from" )
Flavio Castro49c218d2016-06-21 16:55:57 -0700362 main.ONOSbench.cpLogsToDir( "/tmp/karaf.log", main.logdir,
Flavio Castro5608a392016-06-22 17:02:35 -0700363 copyFileName="karaf.log." + main.cfgName )
Flavio Castro49c218d2016-06-21 16:55:57 -0700364 for i in range( main.numCtrls ):
Flavio Castro5608a392016-06-22 17:02:35 -0700365 main.ONOSbench.onosStop( main.ONOSip[ i ] )
Flavio Castro519072e2016-06-23 13:30:48 -0700366
367 @staticmethod
368 def killOnos( main, nodes, switches, links, expNodes ):
369 """
370 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
371 switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
372 Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
373 """
374 main.step( "Killing ONOS instance" )
375 for i in nodes:
376 killResult = main.ONOSbench.onosDie( main.CLIs[ i ].ip_address )
377 utilities.assert_equals( expect=main.TRUE, actual=killResult,
378 onpass="ONOS instance Killed",
379 onfail="Error killing ONOS instance" )
380 if i == main.active:
381 main.active = (i + 1) % main.numCtrls
382 time.sleep( 12 )
383 if len( nodes ) < main.numCtrls:
384 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
385 main.FALSE,
386 kwargs={ 'numoswitch': switches,
387 'numolink': links,
388 'numoctrl': expNodes },
389 attempts=10,
390 sleep=12 )
391 utilities.assert_equals( expect=main.TRUE, actual=topology,
392 onpass="ONOS Instance down successful",
393 onfail="Failed to turn off ONOS Instance" )
394 else:
395 main.active = -1
396
397 @staticmethod
398 def recoverOnos( main, nodes, switches, links, expNodes ):
399 """
400 Params: nodes, integer array with position of the ONOS nodes in the CLIs array
401 switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
402 Recover an ONOS instance and verify the ONOS cluster can see the proper change
403 """
404 main.step( "Recovering ONOS instance" )
405 [ main.ONOSbench.onosStart( main.CLIs[ i ].ip_address ) for i in nodes ]
406 for i in nodes:
407 isUp = main.ONOSbench.isup( main.ONOSip[ i ] )
408 utilities.assert_equals( expect=main.TRUE, actual=isUp,
409 onpass="ONOS service is ready",
410 onfail="ONOS service did not start properly" )
411 for i in nodes:
412 main.step( "Checking if ONOS CLI is ready" )
413 main.CLIs[ i ].startCellCli( )
414 cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
415 commandlineTimeout=60,
416 onosStartTimeout=100 )
417 utilities.assert_equals( expect=main.TRUE,
418 actual=cliResult,
419 onpass="ONOS CLI is ready",
420 onfail="ONOS CLI is not ready" )
421 main.active = i if main.active == -1 else main.active
422
423 topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
424 main.FALSE,
425 kwargs={ 'numoswitch': switches,
426 'numolink': links,
427 'numoctrl': expNodes },
428 attempts=10,
429 sleep=12 )
430 utilities.assert_equals( expect=main.TRUE, actual=topology,
431 onpass="ONOS Instance down successful",
432 onfail="Failed to turn off ONOS Instance" )
433
434 for i in range( 10 ):
435 ready = True
436 output = main.CLIs[ main.active ].summary( )
437 if not output:
438 ready = False
439 if ready:
440 break
441 time.sleep( 10 )
442 utilities.assert_equals( expect=True, actual=ready,
443 onpass="ONOS summary command succeded",
444 onfail="ONOS summary command failed" )
445 if not ready:
446 main.log.error( "ONOS startup failed!" )
447 main.cleanup( )
448 main.exit( )