blob: 201c28cc610a1ca2927d72ff059c52dceec84a68 [file] [log] [blame]
cameron@onlab.us059c2562015-04-02 14:12:51 -07001# ScaleOutTemplate
andrew@onlab.us2ae3a112015-02-02 11:24:32 -08002#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
andrew@onlab.us10332202015-03-11 15:04:43 -07007import sys
cameron@onlab.us059c2562015-04-02 14:12:51 -07008import os.path
cameron@onlab.usaaecfd72015-07-08 12:27:26 -07009import time
andrew@onlab.us10332202015-03-11 15:04:43 -070010
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080011
suibin7f2c9cd2015-07-08 17:34:59 -070012class SCPFintentEventTp:
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080013
14 def __init__( self ):
15 self.default = ''
16
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070017 def CASE1( self, main ):
18 import sys
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070019 import os.path
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070020 import time
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070021
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070022 global init
23 try:
24 if type(init) is not bool:
GlennRCf9ef27d2015-09-16 12:08:38 -070025 init = False
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070026 except NameError:
27 init = False
28
andrew@onlab.us10332202015-03-11 15:04:43 -070029 #Load values from params file
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080030 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
31 gitPull = main.params[ 'GIT' ][ 'autopull' ]
32 cellName = main.params[ 'ENV' ][ 'cellName' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070033 Apps = main.params[ 'ENV' ][ 'cellApps' ]
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080034 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
35 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070036 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070037 maxNodes = int(main.params[ 'max' ])
Jon Hall4ba53f02015-07-29 13:07:41 -070038 main.maxNodes = maxNodes
andrew@onlab.us10332202015-03-11 15:04:43 -070039 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070040 cellName = main.params[ 'ENV' ][ 'cellName' ]
jenkins3af0cd82015-03-24 10:27:16 -070041 numSwitches = (main.params[ 'TEST' ][ 'numSwitches' ]).split(",")
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -070042 flowRuleBU = main.params[ 'TEST' ][ 'flowRuleBUEnabled' ]
suibin zhangd1f03802016-02-25 11:48:58 -080043 skipRelRsrc = main.params[ 'TEST'][ 'skipReleaseResourcesOnWithdrawal']
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070044 homeDir = os.path.expanduser('~')
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080045
cameron@onlab.us412e9562015-05-13 16:04:34 -070046 main.exceptions = [0]*11
47 main.warnings = [0]*11
48 main.errors = [0]*11
49
Jon Hall4ba53f02015-07-29 13:07:41 -070050 # -- INIT SECTION, ONLY RUNS ONCE -- #
51 if init == False:
cameron@onlab.us059c2562015-04-02 14:12:51 -070052 init = True
53 global clusterCount #number of nodes running
54 global ONOSIp #list of ONOS IP addresses
Jon Hall4ba53f02015-07-29 13:07:41 -070055 global scale
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070056 global commit
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070057
cameron@onlab.us059c2562015-04-02 14:12:51 -070058 clusterCount = 0
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070059 ONOSIp = main.ONOSbench.getOnosIps()
60 print ONOSIp
61 print main.ONOSbench.onosIps.values()
62
Jon Hall4ba53f02015-07-29 13:07:41 -070063 scale = (main.params[ 'SCALE' ]).split(",")
cameron@onlab.us059c2562015-04-02 14:12:51 -070064 clusterCount = int(scale[0])
65
Jon Hall4ba53f02015-07-29 13:07:41 -070066 #Populate ONOSIp with ips from params
jenkins15b2b132015-06-23 14:04:09 -070067 ONOSIp.extend(main.ONOSbench.getOnosIps())
jenkins15b2b132015-06-23 14:04:09 -070068
cameron@onlab.us059c2562015-04-02 14:12:51 -070069 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
70 if skipMvn != "yes":
71 mvnResult = main.ONOSbench.cleanInstall()
72
73 #git
74 main.step( "Git checkout and pull " + checkoutBranch )
75 if gitPull == 'on':
76 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
77 pullResult = main.ONOSbench.gitPull()
78
79 else:
80 checkoutResult = main.TRUE
81 pullResult = main.TRUE
82 main.log.info( "Skipped git checkout and pull" )
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070083
Jon Hall4ba53f02015-07-29 13:07:41 -070084 main.log.step("Grabbing commit number")
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070085 commit = main.ONOSbench.getVersion()
86 commit = (commit.split(" "))[1]
Jon Hall4ba53f02015-07-29 13:07:41 -070087
88 main.log.step("Creating results file")
suibin584c0702015-07-14 15:57:27 -070089 resultsDB = open("/tmp/IntentEventTPDB", "w+")
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070090 resultsDB.close()
91
cameron@onlab.us059c2562015-04-02 14:12:51 -070092 # -- END OF INIT SECTION --#
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070093
Jon Hall4ba53f02015-07-29 13:07:41 -070094 main.log.step("Adjusting scale")
95 print str(scale)
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070096 print str(ONOSIp)
cameron@onlab.us059c2562015-04-02 14:12:51 -070097 clusterCount = int(scale[0])
Jon Hall4ba53f02015-07-29 13:07:41 -070098 scale.remove(scale[0])
99
jenkins5a771992015-06-24 13:53:27 -0700100 MN1Ip = ONOSIp[len(ONOSIp) -1]
101 BENCHIp = ONOSIp[len(ONOSIp) -2]
Jon Hall4ba53f02015-07-29 13:07:41 -0700102
103 #kill off all onos processes
andrew@onlab.us10332202015-03-11 15:04:43 -0700104 main.log.step("Safety check, killing all ONOS processes")
Jon Hall70b2ff42015-11-17 15:49:44 -0800105 main.log.step("before initiating environment setup")
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700106 for node in range(maxNodes):
andrew@onlab.us10332202015-03-11 15:04:43 -0700107 main.ONOSbench.onosDie(ONOSIp[node])
Jon Hall4ba53f02015-07-29 13:07:41 -0700108
jenkins8f8f4f32015-06-24 14:05:04 -0700109 MN1Ip = ONOSIp[len(ONOSIp) -1]
110 BENCHIp = ONOSIp[len(ONOSIp) -2]
111
andrew@onlab.us10332202015-03-11 15:04:43 -0700112 #Uninstall everywhere
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800113 main.log.step( "Cleaning Enviornment..." )
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700114 for i in range(maxNodes):
andrew@onlab.us10332202015-03-11 15:04:43 -0700115 main.log.info(" Uninstalling ONOS " + str(i) )
116 main.ONOSbench.onosUninstall( ONOSIp[i] )
suibine1a104d2015-07-08 15:58:29 -0700117 main.log.info("Sleep 10 second for uninstall to settle...")
118 time.sleep(10)
119 main.ONOSbench.handle.sendline(" ")
120 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700121
cameron@onlab.us059c2562015-04-02 14:12:51 -0700122 #construct the cell file
123 main.log.info("Creating cell file")
124 cellIp = []
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700125 for node in range (clusterCount):
cameron@onlab.us059c2562015-04-02 14:12:51 -0700126 cellIp.append(ONOSIp[node])
Jon Hall4ba53f02015-07-29 13:07:41 -0700127
cameron@onlab.usc5bf8192015-07-13 13:36:05 -0700128 main.ONOSbench.createCellFile("localhost",cellName,MN1Ip,str(Apps), cellIp)
cameron@onlab.us059c2562015-04-02 14:12:51 -0700129
130 main.step( "Set Cell" )
131 main.ONOSbench.setCell(cellName)
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800132
jenkins3af0cd82015-03-24 10:27:16 -0700133 myDistribution = []
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700134 for node in range (clusterCount):
135 myDistribution.append(numSwitches[node])
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800136
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800137 main.step( "Creating ONOS package" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700138 packageResult = main.ONOSbench.onosPackage()
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800139
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800140 main.step( "verify cells" )
141 verifyCellResult = main.ONOSbench.verifyCell()
Jon Hall4ba53f02015-07-29 13:07:41 -0700142
cameron@onlab.us059c2562015-04-02 14:12:51 -0700143 main.log.report( "Initializeing " + str( clusterCount ) + " node cluster." )
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700144 for node in range(clusterCount):
cameron@onlab.us059c2562015-04-02 14:12:51 -0700145 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
andrew@onlab.us10332202015-03-11 15:04:43 -0700146 main.ONOSbench.onosInstall( ONOSIp[node])
cameron@onlab.us059c2562015-04-02 14:12:51 -0700147
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700148 for node in range(clusterCount):
andrew@onlab.us10332202015-03-11 15:04:43 -0700149 for i in range( 2 ):
150 isup = main.ONOSbench.isup( ONOSIp[node] )
151 if isup:
152 main.log.info("ONOS " + str(node) + " is up\n")
153 break
154 if not isup:
155 main.log.report( "ONOS " + str(node) + " didn't start!" )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700156 main.log.info("Startup sequence complete")
157
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700158 time.sleep(20)
159
YPZhangf1153fb2016-04-26 10:58:56 -0700160 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled " + str(flowRuleBU))
suibin zhangd1f03802016-02-25 11:48:58 -0800161 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.net.intent.impl.IntentManager", "skipReleaseResourcesOnWithdrawal " + skipRelRsrc)
Jon Hall4ba53f02015-07-29 13:07:41 -0700162 devices = int(clusterCount)*10
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700163
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700164 main.log.step("Setting up null provider")
165 for i in range(3):
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700166 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "deviceCount " + str(devices))
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700167 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "topoShape linear")
168 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "enabled true")
169 time.sleep(5)
170
171 main.ONOSbench.handle.sendline("onos $OC1 summary")
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700172 main.ONOSbench.handle.expect(":~")
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700173
174 before = main.ONOSbench.handle.before
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700175 if ("devices=" + str(devices)) in before:
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700176 break
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700177
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700178 main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
179 main.ONOSbench.handle.expect(":~")
180 print main.ONOSbench.handle.before
You Wang3b148ea2016-06-20 20:33:42 -0700181 time.sleep(5)
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700182
GlennRCf9ef27d2015-09-16 12:08:38 -0700183 for i in range(3):
184 passed = main.ONOSbench.verifySummary( ONOSIp[0] )
185 if passed:
186 main.log.info("Clusters have converged")
andrew@onlab.us10332202015-03-11 15:04:43 -0700187 break
GlennRCf9ef27d2015-09-16 12:08:38 -0700188 else:
189 main.log.error("Clusters have not converged, retying...")
190 time.sleep(3)
191
jenkins50379942015-06-19 13:36:43 -0700192 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
GlennRCf9ef27d2015-09-16 12:08:38 -0700193
Jon Hall4ba53f02015-07-29 13:07:41 -0700194 def CASE2( self, main ):
andrew@onlab.us10332202015-03-11 15:04:43 -0700195 import time
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800196 import json
cameron@onlab.us059c2562015-04-02 14:12:51 -0700197 import string
andrew@onlab.us10332202015-03-11 15:04:43 -0700198 import csv
jenkins3af0cd82015-03-24 10:27:16 -0700199 import numpy
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700200 import os.path
201
202 global currentNeighbors
203 neighbors = []
204
205 try:
206 currentNeighbors
207 except:
208 currentNeighbors = "0"
209 neighbors = ['0']
210 else:
211 if currentNeighbors == "r": #reset
212 currentNeighbors = "a"
213 neighbors = ['0']
214 else:
Jon Hall4ba53f02015-07-29 13:07:41 -0700215 currentNeighbors = "r"
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700216 neighbors = ['a']
217
218 if clusterCount == 1:
219 currentNeighbors = "r"
andrew@onlab.us10332202015-03-11 15:04:43 -0700220
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800221 main.log.info("Cluster Count = " + str(clusterCount))
222
andrew@onlab.us10332202015-03-11 15:04:43 -0700223 intentsRate = main.params['METRICS']['intents_rate']
224 intentsWithdrawn = main.params[ 'METRICS' ][ 'intents_withdrawn' ]
225 intentsFailed = main.params[ 'METRICS' ][ 'intents_failed' ]
226 testDuration = main.params[ 'TEST' ][ 'duration' ]
227 logInterval = main.params[ 'TEST' ][ 'log_interval' ]
228 debug = main.params[ 'debugMode' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -0700229 numKeys = main.params[ 'TEST' ][ 'numKeys' ]
230 cyclePeriod = main.params[ 'TEST' ][ 'cyclePeriod' ]
Jon Hall4ba53f02015-07-29 13:07:41 -0700231 #neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",")
andrew@onlab.us10332202015-03-11 15:04:43 -0700232 metricList = [intentsRate, intentsWithdrawn, intentsFailed]
jenkins3af0cd82015-03-24 10:27:16 -0700233
Jon Hall4ba53f02015-07-29 13:07:41 -0700234 for n in range(0, len(neighbors)):
235 if neighbors[n] == 'a':
cameron@onlab.us059c2562015-04-02 14:12:51 -0700236 neighbors[n] = str(clusterCount -1)
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700237 if int(clusterCount) == 1:
238 neighbors = neighbors.pop()
Jon Hall4ba53f02015-07-29 13:07:41 -0700239
cameron@onlab.us059c2562015-04-02 14:12:51 -0700240 for n in neighbors:
Jon Hall4ba53f02015-07-29 13:07:41 -0700241 main.log.info("Run with " + n + " neighbors")
cameron@onlab.us059c2562015-04-02 14:12:51 -0700242 time.sleep(5)
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700243 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numKeys " + numKeys )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700244 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700245 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numNeighbors " + n )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700246 main.ONOSbench.handle.expect(":~")
247 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller cyclePeriod " + cyclePeriod )
248 main.ONOSbench.handle.expect(":~")
jenkins3af0cd82015-03-24 10:27:16 -0700249
cameron@onlab.us059c2562015-04-02 14:12:51 -0700250 cmd = "onos $OC1 intent-perf-start"
251 main.ONOSbench.handle.sendline(cmd)
252 main.ONOSbench.handle.expect(":~")
253 main.log.info("Starting ONOS (all nodes) intent-perf from $OC1" )
jenkins3af0cd82015-03-24 10:27:16 -0700254
cameron@onlab.us059c2562015-04-02 14:12:51 -0700255 main.log.info( "Starting test loop for " + str(testDuration) + " seconds...\n" )
256 stop = time.time() + float( testDuration )
jenkins3af0cd82015-03-24 10:27:16 -0700257
cameron@onlab.us059c2562015-04-02 14:12:51 -0700258 while time.time() < stop:
259 time.sleep( float( logInterval ) )
260 groupResult = []
261 for node in range (1, clusterCount + 1):
262 groupResult.append(0)
jenkins3af0cd82015-03-24 10:27:16 -0700263
GlennRCb0773152015-09-13 21:02:11 -0700264 cmd = " onos-ssh $OC" + str(node) + """ cat /opt/onos/log/karaf.log | grep "Throughput:" | tail -1 """
cameron@onlab.us059c2562015-04-02 14:12:51 -0700265 main.log.info("COMMAND: " + str(cmd))
266
267 x = 0
268 while True:
269 main.ONOSbench.handle.sendline(cmd)
jenkins50379942015-06-19 13:36:43 -0700270 time.sleep(6)
cameron@onlab.us059c2562015-04-02 14:12:51 -0700271 main.ONOSbench.handle.expect(":~")
272 raw = main.ONOSbench.handle.before
273 if "OVERALL=" in raw:
274 break
275 x += 1
276 if x > 10:
277 main.log.error("Expected output not being recieved... continuing")
278 break
279 time.sleep(2)
280
281 raw = raw.splitlines()
282 splitResults = []
283 for line in raw:
284 splitResults.extend(line.split(" "))
285
286 myResult = "--"
287 for field in splitResults:
288 if "OVERALL" in field:
289 myResult = field
290
291 if myResult == "--":
292 main.log.error("Parsing/Pexpect error\n" + str(splitResults))
293
294 myResult = myResult.replace(";", "")
295 myResult = myResult.replace("OVERALL=","")
296 myResult = float(myResult)
297 groupResult[len(groupResult) -1] = myResult
298
299 main.log.info("Node " + str(node) + " overall rate: " + str(myResult))
300
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700301 clusterTotal = str(numpy.sum(groupResult))
cameron@onlab.us059c2562015-04-02 14:12:51 -0700302 main.log.report("Results from this round of polling: " + str(groupResult))
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700303 main.log.report("Cluster Total: " + clusterTotal + "\n")
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700304
cameron@onlab.us059c2562015-04-02 14:12:51 -0700305 cmd = "onos $OC1 intent-perf-stop"
306 main.ONOSbench.handle.sendline(cmd)
307 main.ONOSbench.handle.expect(":~")
308 main.log.info("Stopping intentperf" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700309
GlennRC47736b82015-09-16 14:27:51 -0700310 with open("/tmp/IntentEventTPDB", "a") as resultsDB:
311 for node in groupResult:
312 resultString = "'" + commit + "',"
313 resultString += "'1gig',"
314 resultString += str(clusterCount) + ","
315 resultString += "'baremetal" + str(int(groupResult.index(node)) + 1) + "',"
316 resultString += n + ","
317 resultString += str(node) + ","
318 resultString += str(0) + "\n" #no stddev
319 resultsDB.write(resultString)
Jon Hall4ba53f02015-07-29 13:07:41 -0700320
321 resultsDB.close()
322
323 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700324