blob: 03d6c567dad35eafd88935d9d2ebf8dd28ce755e [file] [log] [blame]
YPZhangcb86c5b2016-01-27 17:38:12 -08001import sys
2import json
3import time
4import os
5'''
6SCPFscalingMaxIntents
7Push test Intents to onos
8CASE10: set up Null Provider
9CASE11: set up Open Flows
10Scale up when reach the Limited
11Start from 1 nodes, 8 devices. Then Scale up to 3,5,7 nodes
12Recommand batch size: 100, check interval: 100
13'''
14class SCPFscalingMaxIntents:
15 def __init__( self ):
16 self.default = ''
17
18 def CASE0( self, main):
19 import sys
20 import json
21 import time
22 import os
23 import imp
24
25 main.case( "Constructing test variables and building ONOS package" )
26 main.step( "Constructing test variables" )
27 stepResult = main.FALSE
28
29 # Test variables
30 main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
31 main.dependencyPath = main.testOnDirectory + \
32 main.params['DEPENDENCY']['path']
33 main.cellName = main.params[ 'ENV' ][ 'cellName' ]
34 main.apps = main.params[ 'ENV' ][ 'cellApps' ]
35 main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
36 main.scale = ( main.params[ 'SCALE' ] ).split( "," )
37 main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
38 main.timeout = int(main.params['SLEEP']['timeout'])
39 main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
40 main.installSleep = int( main.params[ 'SLEEP' ][ 'install' ] )
41 main.verifySleep = int( main.params[ 'SLEEP' ][ 'verify' ] )
42 main.rerouteSleep = int ( main.params['SLEEP']['reroute'] )
43 main.verifyAttempts = int( main.params['ATTEMPTS']['verify'] )
44 main.ingress = main.params['LINK']['ingress']
45 main.egress = main.params['LINK']['egress']
46 main.dbFileName = main.params['DATABASE']['file']
47 main.cellData = {} # for creating cell file
48 main.reroute = main.params['reroute']
49 main.threadID = 0
50
51 if main.reroute == "True":
52 main.reroute = True
53 else:
54 main.reroute = False
55
56 main.CLIs = []
57 main.ONOSip = []
58 main.maxNumBatch = 0
59 main.ONOSip = main.ONOSbench.getOnosIps()
60 main.log.info(main.ONOSip)
61 main.setupSkipped = False
62
63 wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
64 gitBranch = main.params[ 'GIT' ][ 'branch' ]
65 gitPull = main.params[ 'GIT' ][ 'pull' ]
66 nic = main.params['DATABASE']['nic']
67 node = main.params['DATABASE']['node']
68 nic = main.params['DATABASE']['nic']
69 node = main.params['DATABASE']['node']
70 stepResult = main.TRUE
71
72 main.log.info("Cresting DB file")
73 with open(main.dbFileName, "w+") as dbFile:
74 dbFile.write("")
75
76 utilities.assert_equals( expect=main.TRUE,
77 actual=stepResult,
78 onpass="environment set up successfull",
79 onfail="environment set up Failed" )
80
81 def CASE1( self ):
82 # main.scale[ 0 ] determines the current number of ONOS controller
83 main.CLIs = []
84 main.numCtrls = int( main.scale[ 0 ] )
85 main.log.info( "Creating list of ONOS cli handles" )
86 for i in range(main.numCtrls):
87 main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
88
89 main.log.info(main.CLIs)
90 if not main.CLIs:
91 main.log.error( "Failed to create the list of ONOS cli handles" )
92 main.cleanup()
93 main.exit()
94
95 main.log.info( "Loading wrapper files" )
96 main.startUp = imp.load_source( wrapperFile1,
97 main.dependencyPath +
98 wrapperFile1 +
99 ".py" )
100
101 copyResult = main.ONOSbench.copyMininetFile( main.topology,
102 main.dependencyPath,
103 main.Mininet1.user_name,
104 main.Mininet1.ip_address )
105
106 commit = main.ONOSbench.getVersion(report=True)
107 commit = commit.split(" ")[1]
108
109 if gitPull == 'True':
110 if not main.startUp.onosBuild( main, gitBranch ):
111 main.log.error( "Failed to build ONOS" )
112 main.cleanup()
113 main.exit()
114 else:
115 main.log.warn( "Did not pull new code so skipping mvn " +
116 "clean install" )
117 with open(main.dbFileName, "a") as dbFile:
118 temp = "'" + commit + "',"
119 temp += "'" + nic + "',"
120 dbFile.write(temp)
121
122 def CASE2( self, main ):
123 """
124 - Uninstall ONOS cluster
125 - Verify ONOS start up
126 - Install ONOS cluster
127 - Connect to cli
128 """
129 main.log.info( "Starting up %s node(s) ONOS cluster" % main.numCtrls)
130 main.log.info( "Safety check, killing all ONOS processes" +
131 " before initiating environment setup" )
132
133 for i in range( main.numCtrls ):
134 main.ONOSbench.onosDie( main.ONOSip[ i ] )
135
136 main.log.info( "NODE COUNT = %s" % main.numCtrls)
137
138 tempOnosIp = []
139 for i in range( main.numCtrls ):
140 tempOnosIp.append( main.ONOSip[i] )
141
142 main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
143 "temp",
144 main.Mininet1.ip_address,
145 main.apps,
146 tempOnosIp )
147
148 main.step( "Apply cell to environment" )
149 cellResult = main.ONOSbench.setCell( "temp" )
150 verifyResult = main.ONOSbench.verifyCell()
151 stepResult = cellResult and verifyResult
152 utilities.assert_equals( expect=main.TRUE,
153 actual=stepResult,
154 onpass="Successfully applied cell to " + \
155 "environment",
156 onfail="Failed to apply cell to environment " )
157
158 main.step( "Creating ONOS package" )
159 packageResult = main.ONOSbench.onosPackage()
160 stepResult = packageResult
161 utilities.assert_equals( expect=main.TRUE,
162 actual=stepResult,
163 onpass="Successfully created ONOS package",
164 onfail="Failed to create ONOS package" )
165
166 main.step( "Uninstall ONOS package on all Nodes" )
167 uninstallResult = main.TRUE
168 for i in range( int( main.numCtrls ) ):
169 main.log.info( "Uninstalling package on ONOS Node IP: " + main.ONOSip[i] )
170 u_result = main.ONOSbench.onosUninstall( main.ONOSip[i] )
171 utilities.assert_equals( expect=main.TRUE, actual=u_result,
172 onpass="Test step PASS",
173 onfail="Test step FAIL" )
174 uninstallResult = ( uninstallResult and u_result )
175
176 main.step( "Install ONOS package on all Nodes" )
177 installResult = main.TRUE
178 for i in range( int( main.numCtrls ) ):
179 main.log.info( "Installing package on ONOS Node IP: " + main.ONOSip[i] )
180 i_result = main.ONOSbench.onosInstall( node=main.ONOSip[i] )
181 utilities.assert_equals( expect=main.TRUE, actual=i_result,
182 onpass="Test step PASS",
183 onfail="Test step FAIL" )
184 installResult = installResult and i_result
185
186 main.step( "Verify ONOS nodes UP status" )
187 statusResult = main.TRUE
188 for i in range( int( main.numCtrls ) ):
189 main.log.info( "ONOS Node " + main.ONOSip[i] + " status:" )
190 onos_status = main.ONOSbench.onosStatus( node=main.ONOSip[i] )
191 utilities.assert_equals( expect=main.TRUE, actual=onos_status,
192 onpass="Test step PASS",
193 onfail="Test step FAIL" )
194 statusResult = ( statusResult and onos_status )
195
196 main.step( "Start ONOS CLI on all nodes" )
197 cliResult = main.TRUE
198 main.log.step(" Start ONOS cli using thread ")
199 startCliResult = main.TRUE
200 pool = []
201
202 for i in range( int( main.numCtrls) ):
203 t = main.Thread( target=main.CLIs[i].startOnosCli,
204 threadID=main.threadID,
205 name="startOnosCli",
206 args=[ main.ONOSip[i] ],
207 kwargs = {"onosStartTimeout":main.timeout} )
208 pool.append(t)
209 t.start()
210 main.threadID = main.threadID + 1
211 for t in pool:
212 t.join()
213 startCliResult = startCliResult and t.result
214 time.sleep( main.startUpSleep )
215
216 def CASE10( self, main ):
217 """
218 Setting up null-provider
219 """
220 import json
221 # Activate apps
222 main.step("Activating null-provider")
223 appStatus = utilities.retry( main.CLIs[0].activateApp,
224 main.FALSE,
225 ['org.onosproject.null'],
226 sleep=main.verifySleep,
227 attempts=main.verifyAttempts )
228 utilities.assert_equals( expect=main.TRUE,
229 actual=appStatus,
230 onpass="Successfully activated null-provider",
231 onfail="Failed activate null-provider" )
232
233 # Setup the null-provider
234 main.step("Configuring null-provider")
235 cfgStatus = utilities.retry( main.ONOSbench.onosCfgSet,
236 main.FALSE,
237 [ main.ONOSip[0],
238 'org.onosproject.provider.nil.NullProviders', 'deviceCount 8'],
239 sleep=main.verifySleep,
240 attempts = main.verifyAttempts )
241 cfgStatus = cfgStatus and utilities.retry( main.ONOSbench.onosCfgSet,
242 main.FALSE,
243 [ main.ONOSip[0],
244 'org.onosproject.provider.nil.NullProviders', 'topoShape reroute'],
245 sleep=main.verifySleep,
246 attempts = main.verifyAttempts )
247
248 cfgStatus = cfgStatus and utilities.retry( main.ONOSbench.onosCfgSet,
249 main.FALSE,
250 [ main.ONOSip[0],
251 'org.onosproject.provider.nil.NullProviders', 'enabled true'],
252 sleep=main.verifySleep,
253 attempts = main.verifyAttempts )
254
255
256 utilities.assert_equals( expect=main.TRUE,
257 actual=cfgStatus,
258 onpass="Successfully configured null-provider",
259 onfail="Failed to configure null-provider" )
260
261 # give onos some time to settle
262 time.sleep(main.startUpSleep)
263
264 main.log.info("Setting default flows to zero")
265 main.defaultFlows = 0
266
267 main.step("Check status of null-provider setup")
268 caseResult = appStatus and cfgStatus
269 utilities.assert_equals( expect=main.TRUE,
270 actual=caseResult,
271 onpass="Setting up null-provider was successfull",
272 onfail="Failed to setup null-provider" )
273
274 # This tells the following cases if we are using the null-provider or ovs
275 main.switchType = "null:"
276
277 # If the null-provider setup was unsuccessfull, then there is no point to
278 # run the subsequent cases
279
280 time.sleep(main.startUpSleep)
281 main.step( "Balancing Masters" )
282
283 stepResult = main.FALSE
284 stepResult = utilities.retry( main.CLIs[0].balanceMasters,
285 main.FALSE,
286 [],
287 sleep=3,
288 attempts=3 )
289
290 utilities.assert_equals( expect=main.TRUE,
291 actual=stepResult,
292 onpass="Balance masters was successfull",
293 onfail="Failed to balance masters")
294
295 time.sleep( 5 )
296 if not caseResult:
297 main.setupSkipped = True
298
299 def CASE11( self, main):
300 '''
301 Setting up mininet
302 '''
303 import json
304 import time
305
306 time.sleep(main.startUpSleep)
307
308 main.step("Activating openflow")
309 appStatus = utilities.retry( main.ONOSrest1.activateApp,
310 main.FALSE,
311 ['org.onosproject.openflow'],
312 sleep=3,
313 attempts=3 )
314 utilities.assert_equals( expect=main.TRUE,
315 actual=appStatus,
316 onpass="Successfully activated openflow",
317 onfail="Failed activate openflow" )
318
319 time.sleep(main.startUpSleep)
320 main.step('Starting mininet topology')
321 mnStatus = main.Mininet1.startNet(topoFile='~/mininet/custom/rerouteTopo.py')
322 utilities.assert_equals( expect=main.TRUE,
323 actual=mnStatus,
324 onpass="Successfully started Mininet",
325 onfail="Failed to activate Mininet" )
326
327 main.step("Assinging masters to switches")
328 switches = main.Mininet1.getSwitches()
329 swStatus = main.Mininet1.assignSwController( sw=switches.keys(), ip=main.ONOSip )
330 utilities.assert_equals( expect=main.TRUE,
331 actual=swStatus,
332 onpass="Successfully assigned switches to masters",
333 onfail="Failed assign switches to masters" )
334
335 time.sleep(main.startUpSleep)
336
337 main.log.info("Getting default flows")
338 jsonSum = json.loads(main.CLIs[0].summary())
339 main.defaultFlows = jsonSum["flows"]
340
341 main.step("Check status of Mininet setup")
342 caseResult = appStatus and mnStatus and swStatus
343 utilities.assert_equals( expect=main.TRUE,
344 actual=caseResult,
345 onpass="Successfully setup Mininet",
346 onfail="Failed setup Mininet" )
347
348 # This tells the following cases if we are using the null-provider or ovs
349 main.switchType = "of:"
350
351 time.sleep(main.startUpSleep)
352 main.step( "Balancing Masters" )
353
354 stepResult = main.FALSE
355 stepResult = utilities.retry( main.CLIs[0].balanceMasters,
356 main.FALSE,
357 [],
358 sleep=3,
359 attempts=3 )
360
361 utilities.assert_equals( expect=main.TRUE,
362 actual=stepResult,
363 onpass="Balance masters was successfull",
364 onfail="Failed to balance masters")
365
366 time.sleep(5)
367 if not caseResult:
368 main.setupSkipped = True
369
370
371
372 def CASE20( self, main ):
373 if main.reroute:
374 main.minIntents = int(main.params['NULL']['REROUTE']['min_intents'])
375 main.maxIntents = int(main.params['NULL']['REROUTE']['max_intents'])
376 main.checkInterval = int(main.params['NULL']['REROUTE']['check_interval'])
377 main.batchSize = int(main.params['NULL']['REROUTE']['batch_size'])
378 else:
379 main.minIntents = int(main.params['NULL']['PUSH']['min_intents'])
380 main.maxIntents = int(main.params['NULL']['PUSH']['max_intents'])
381 main.checkInterval = int(main.params['NULL']['PUSH']['check_interval'])
382 main.batchSize = int(main.params['NULL']['PUSH']['batch_size'])
383
384 # check if the case needs to be skipped
385 if main.setupSkipped:
386 main.setupSkipped = False
387 main.skipCase()
388
389 # the index where the next intents will be installed
390 offfset = 0
391 # keeps track of how many intents have been installed
392 currIntents = 0
YPZhang3097ba92016-02-16 17:32:32 -0800393 # keeps track of how many flows have been installed, set to 0 at start
394 currFlows = 0
YPZhangcb86c5b2016-01-27 17:38:12 -0800395 # limit for the number of intents that can be installed
396 limit = main.maxIntents / main.batchSize
397 # total intents installed
398 totalIntents = 0
399
400 intentsState = None
401
402 offtmp = 0
403 main.step( "Pushing intents" )
404 stepResult = main.TRUE
405
406 for i in range(limit):
407
408 # Threads pool
409 pool = []
410
411 for j in range( int( main.numCtrls) ):
412 if main.numCtrls > 1:
413 time.sleep( 1 )
414 offtmp = offfset + main.maxIntents * j
415 # Push intents by using threads
416 t = main.Thread( target=main.CLIs[j].pushTestIntents,
417 threadID=main.threadID,
418 name="Push-Test-Intents",
419 args=[ main.switchType + main.ingress,
420 main.switchType + main.egress,
421 main.batchSize ],
422 kwargs={ "offset": offtmp,
423 "options": "-i",
424 "timeout": main.timeout,
425 "background":False } )
426 pool.append(t)
427 t.start()
428 main.threadID = main.threadID + 1
429 for t in pool:
430 t.join()
431 stepResult = stepResult and t.result
432 offfset = offfset + main.batchSize
433
YPZhang3097ba92016-02-16 17:32:32 -0800434 totalIntents = main.batchSize * main.numCtrls + totalIntents
YPZhangcb86c5b2016-01-27 17:38:12 -0800435 if totalIntents >= main.minIntents and totalIntents % main.checkInterval == 0:
436 # if reach to minimum number and check interval, verify Intetns and flows
437 time.sleep( main.verifySleep * main.numCtrls )
438
439 main.log.info("Verify Intents states")
YPZhang3097ba92016-02-16 17:32:32 -0800440 # k is a control variable for verify retry attempts
YPZhangcb86c5b2016-01-27 17:38:12 -0800441 k = 1
442 intentVerify = main.FALSE
443
YPZhang3097ba92016-02-16 17:32:32 -0800444 while k <= main.verifyAttempts:
YPZhangcb86c5b2016-01-27 17:38:12 -0800445 # while loop for check intents by using REST api
446 time.sleep(5)
447 temp = 0
448 intentsState = json.loads( main.ONOSrest1.intents() )
449 for f in intentsState:
450 # get INSTALLED intents number
451 if f.get("state") == "INSTALLED":
452 temp = temp + 1
453
YPZhangcb86c5b2016-01-27 17:38:12 -0800454 main.log.info("Total Intents: {} INSTALLED: {}".format(totalIntents, temp))
455 if totalIntents == temp:
456 intentVerify = main.TRUE
457 break
458 intentVerify = main.FALSE
459 k = k+1
460
461 if not intentVerify:
462 # If some intents are not installed, finished this test case
463 main.log.warn( "Some intens did not install" )
464 # We don't want to check flows if intents not installed, because onos will drop flows
YPZhang3097ba92016-02-16 17:32:32 -0800465 if currFlows == 0:
466 # If currFlows equal 0, which means we failed to install intents at first, or we didn't get
467 # the correct number, so we need get flows here.
468 flowsState = json.loads( main.ONOSrest1.flows() )
YPZhangcb86c5b2016-01-27 17:38:12 -0800469 break
470
471 main.log.info("Verify Flows states")
472 k = 1
473 flowsVerify = main.TRUE
474
YPZhang3097ba92016-02-16 17:32:32 -0800475 while k <= main.verifyAttempts:
476 # while loop for check flows by using REST api
YPZhangcb86c5b2016-01-27 17:38:12 -0800477 time.sleep(3)
478 temp = 0
479 flowsStateCount = []
480 flowsState = json.loads( main.ONOSrest1.flows() )
481 for f in flowsState:
482 # get PENDING_ADD flows
483 if f.get("state") == "PENDING_ADD":
484 temp = temp + 1
485
486 flowsStateCount.append(temp)
487 temp = 0
488
489 for f in flowsState:
490 # get PENDING_REMOVE flows
491 if f.get("state") == "PENDING_REMOVE":
492 temp = temp + 1
493
494 flowsStateCount.append(temp)
495 temp = 0
496
497 for f in flowsState:
498 # get REMOVED flows
499 if f.get("state") == "REMOVED":
500 temp = temp + 1
501
502 flowsStateCount.append(temp)
503 temp = 0
504
505 for f in flowsState:
506 # get FAILED flwos
507 if f.get("state") == "FAILED":
508 temp = temp + 1
509
510 flowsStateCount.append(temp)
511 temp = 0
512 k = k + 1
513 for c in flowsStateCount:
514 if int(c) > 0:
515 flowsVerify = main.FALSE
516
517 main.log.info( "Check flows States:" )
518 main.log.info( "PENDING_ADD: {}".format( flowsStateCount[0]) )
519 main.log.info( "PENDING_REMOVE: {}".format( flowsStateCount[1]) )
520 main.log.info( "REMOVED: {}".format( flowsStateCount[2]) )
521 main.log.info( "FAILED: {}".format( flowsStateCount[3]) )
522
523 if flowsVerify == main.TRUE:
524 break
525
526 del main.scale[0]
527 utilities.assert_equals( expect = main.TRUE,
528 actual = intentVerify,
529 onpass = "Successfully pushed and verified intents",
530 onfail = "Failed to push and verify intents" )
531
532 # we need the total intents before crash
533 totalIntents = len(intentsState)
534 totalFlows = len(flowsState)
535
536 main.log.info( "Total Intents Installed before crash: {}".format( totalIntents ) )
537 main.log.info( "Total Flows ADDED before crash: {}".format( totalFlows ) )
538
539 main.step('clean up Mininet')
540 main.Mininet1.stopNet()
541
542 main.log.info("Writing results to DS file")
543 with open(main.dbFileName, "a") as dbFile:
544 # Scale number
545 temp = str(main.numCtrls)
546 temp += ",'" + "baremetal1" + "'"
547 # how many intents we installed before crash
548 temp += "," + str(totalIntents)
549 # how many flows we installed before crash
550 temp += "," + str(totalFlows)
551 # other columns in database, but we didn't use in this test
552 temp += "," + "0,0,0,0,0,0"
553 temp += "\n"
554 dbFile.write( temp )