blob: e017a489750cf73319375a84407e5d086488cd32 [file] [log] [blame]
cameron@onlab.us059c2562015-04-02 14:12:51 -07001# ScaleOutTemplate
andrew@onlab.us2ae3a112015-02-02 11:24:32 -08002#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
andrew@onlab.us10332202015-03-11 15:04:43 -07007import sys
cameron@onlab.us059c2562015-04-02 14:12:51 -07008import os.path
cameron@onlab.usaaecfd72015-07-08 12:27:26 -07009import time
andrew@onlab.us10332202015-03-11 15:04:43 -070010
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080011
suibin7f2c9cd2015-07-08 17:34:59 -070012class SCPFintentEventTp:
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080013
14 def __init__( self ):
15 self.default = ''
16
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070017 def CASE1( self, main ):
18 import sys
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070019 import os.path
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070020 import time
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070021
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070022 global init
23 try:
24 if type(init) is not bool:
GlennRCf9ef27d2015-09-16 12:08:38 -070025 init = False
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070026 except NameError:
27 init = False
28
andrew@onlab.us10332202015-03-11 15:04:43 -070029 #Load values from params file
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080030 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
31 gitPull = main.params[ 'GIT' ][ 'autopull' ]
32 cellName = main.params[ 'ENV' ][ 'cellName' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070033 Apps = main.params[ 'ENV' ][ 'cellApps' ]
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080034 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
35 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070036 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070037 maxNodes = int(main.params[ 'max' ])
Jon Hall4ba53f02015-07-29 13:07:41 -070038 main.maxNodes = maxNodes
andrew@onlab.us10332202015-03-11 15:04:43 -070039 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070040 cellName = main.params[ 'ENV' ][ 'cellName' ]
jenkins3af0cd82015-03-24 10:27:16 -070041 numSwitches = (main.params[ 'TEST' ][ 'numSwitches' ]).split(",")
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -070042 flowRuleBU = main.params[ 'TEST' ][ 'flowRuleBUEnabled' ]
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070043 homeDir = os.path.expanduser('~')
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080044
cameron@onlab.us412e9562015-05-13 16:04:34 -070045 main.exceptions = [0]*11
46 main.warnings = [0]*11
47 main.errors = [0]*11
48
Jon Hall4ba53f02015-07-29 13:07:41 -070049 # -- INIT SECTION, ONLY RUNS ONCE -- #
50 if init == False:
cameron@onlab.us059c2562015-04-02 14:12:51 -070051 init = True
52 global clusterCount #number of nodes running
53 global ONOSIp #list of ONOS IP addresses
Jon Hall4ba53f02015-07-29 13:07:41 -070054 global scale
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070055 global commit
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070056
cameron@onlab.us059c2562015-04-02 14:12:51 -070057 clusterCount = 0
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070058 ONOSIp = main.ONOSbench.getOnosIps()
59 print ONOSIp
60 print main.ONOSbench.onosIps.values()
61
Jon Hall4ba53f02015-07-29 13:07:41 -070062 scale = (main.params[ 'SCALE' ]).split(",")
cameron@onlab.us059c2562015-04-02 14:12:51 -070063 clusterCount = int(scale[0])
64
Jon Hall4ba53f02015-07-29 13:07:41 -070065 #Populate ONOSIp with ips from params
jenkins15b2b132015-06-23 14:04:09 -070066 ONOSIp.extend(main.ONOSbench.getOnosIps())
jenkins15b2b132015-06-23 14:04:09 -070067
cameron@onlab.us059c2562015-04-02 14:12:51 -070068 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
69 if skipMvn != "yes":
70 mvnResult = main.ONOSbench.cleanInstall()
71
72 #git
73 main.step( "Git checkout and pull " + checkoutBranch )
74 if gitPull == 'on':
75 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
76 pullResult = main.ONOSbench.gitPull()
77
78 else:
79 checkoutResult = main.TRUE
80 pullResult = main.TRUE
81 main.log.info( "Skipped git checkout and pull" )
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070082
Jon Hall4ba53f02015-07-29 13:07:41 -070083 main.log.step("Grabbing commit number")
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070084 commit = main.ONOSbench.getVersion()
85 commit = (commit.split(" "))[1]
Jon Hall4ba53f02015-07-29 13:07:41 -070086
87 main.log.step("Creating results file")
suibin584c0702015-07-14 15:57:27 -070088 resultsDB = open("/tmp/IntentEventTPDB", "w+")
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070089 resultsDB.close()
90
cameron@onlab.us059c2562015-04-02 14:12:51 -070091 # -- END OF INIT SECTION --#
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070092
Jon Hall4ba53f02015-07-29 13:07:41 -070093 main.log.step("Adjusting scale")
94 print str(scale)
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070095 print str(ONOSIp)
cameron@onlab.us059c2562015-04-02 14:12:51 -070096 clusterCount = int(scale[0])
Jon Hall4ba53f02015-07-29 13:07:41 -070097 scale.remove(scale[0])
98
jenkins5a771992015-06-24 13:53:27 -070099 MN1Ip = ONOSIp[len(ONOSIp) -1]
100 BENCHIp = ONOSIp[len(ONOSIp) -2]
Jon Hall4ba53f02015-07-29 13:07:41 -0700101
102 #kill off all onos processes
andrew@onlab.us10332202015-03-11 15:04:43 -0700103 main.log.step("Safety check, killing all ONOS processes")
Jon Hall70b2ff42015-11-17 15:49:44 -0800104 main.log.step("before initiating environment setup")
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700105 for node in range(maxNodes):
andrew@onlab.us10332202015-03-11 15:04:43 -0700106 main.ONOSbench.onosDie(ONOSIp[node])
Jon Hall4ba53f02015-07-29 13:07:41 -0700107
jenkins8f8f4f32015-06-24 14:05:04 -0700108 MN1Ip = ONOSIp[len(ONOSIp) -1]
109 BENCHIp = ONOSIp[len(ONOSIp) -2]
110
andrew@onlab.us10332202015-03-11 15:04:43 -0700111 #Uninstall everywhere
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800112 main.log.step( "Cleaning Enviornment..." )
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700113 for i in range(maxNodes):
andrew@onlab.us10332202015-03-11 15:04:43 -0700114 main.log.info(" Uninstalling ONOS " + str(i) )
115 main.ONOSbench.onosUninstall( ONOSIp[i] )
suibine1a104d2015-07-08 15:58:29 -0700116 main.log.info("Sleep 10 second for uninstall to settle...")
117 time.sleep(10)
118 main.ONOSbench.handle.sendline(" ")
119 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700120
cameron@onlab.us059c2562015-04-02 14:12:51 -0700121 #construct the cell file
122 main.log.info("Creating cell file")
123 cellIp = []
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700124 for node in range (clusterCount):
cameron@onlab.us059c2562015-04-02 14:12:51 -0700125 cellIp.append(ONOSIp[node])
Jon Hall4ba53f02015-07-29 13:07:41 -0700126
cameron@onlab.usc5bf8192015-07-13 13:36:05 -0700127 main.ONOSbench.createCellFile("localhost",cellName,MN1Ip,str(Apps), cellIp)
cameron@onlab.us059c2562015-04-02 14:12:51 -0700128
129 main.step( "Set Cell" )
130 main.ONOSbench.setCell(cellName)
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800131
jenkins3af0cd82015-03-24 10:27:16 -0700132 myDistribution = []
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700133 for node in range (clusterCount):
134 myDistribution.append(numSwitches[node])
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800135
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800136 main.step( "Creating ONOS package" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700137 packageResult = main.ONOSbench.onosPackage()
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800138
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800139 main.step( "verify cells" )
140 verifyCellResult = main.ONOSbench.verifyCell()
Jon Hall4ba53f02015-07-29 13:07:41 -0700141
cameron@onlab.us059c2562015-04-02 14:12:51 -0700142 main.log.report( "Initializeing " + str( clusterCount ) + " node cluster." )
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700143 for node in range(clusterCount):
cameron@onlab.us059c2562015-04-02 14:12:51 -0700144 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
andrew@onlab.us10332202015-03-11 15:04:43 -0700145 main.ONOSbench.onosInstall( ONOSIp[node])
cameron@onlab.us059c2562015-04-02 14:12:51 -0700146
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700147 for node in range(clusterCount):
andrew@onlab.us10332202015-03-11 15:04:43 -0700148 for i in range( 2 ):
149 isup = main.ONOSbench.isup( ONOSIp[node] )
150 if isup:
151 main.log.info("ONOS " + str(node) + " is up\n")
152 break
153 if not isup:
154 main.log.report( "ONOS " + str(node) + " didn't start!" )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700155 main.log.info("Startup sequence complete")
156
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700157 time.sleep(20)
158
suibin zhang0b9346b2015-09-03 12:22:52 -0700159 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.store.flow.impl.NewDistributedFlowRuleStore", "backupEnabled " + str(flowRuleBU))
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700160
Jon Hall4ba53f02015-07-29 13:07:41 -0700161 devices = int(clusterCount)*10
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700162
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700163 main.log.step("Setting up null provider")
164 for i in range(3):
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700165 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "deviceCount " + str(devices))
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700166 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "topoShape linear")
167 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "enabled true")
168 time.sleep(5)
169
170 main.ONOSbench.handle.sendline("onos $OC1 summary")
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700171 main.ONOSbench.handle.expect(":~")
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700172
173 before = main.ONOSbench.handle.before
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700174 if ("devices=" + str(devices)) in before:
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700175 break
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700176
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700177 main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
178 main.ONOSbench.handle.expect(":~")
179 print main.ONOSbench.handle.before
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700180
GlennRCf9ef27d2015-09-16 12:08:38 -0700181 for i in range(3):
182 passed = main.ONOSbench.verifySummary( ONOSIp[0] )
183 if passed:
184 main.log.info("Clusters have converged")
andrew@onlab.us10332202015-03-11 15:04:43 -0700185 break
GlennRCf9ef27d2015-09-16 12:08:38 -0700186 else:
187 main.log.error("Clusters have not converged, retying...")
188 time.sleep(3)
189
jenkins50379942015-06-19 13:36:43 -0700190 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
GlennRCf9ef27d2015-09-16 12:08:38 -0700191
Jon Hall4ba53f02015-07-29 13:07:41 -0700192 def CASE2( self, main ):
andrew@onlab.us10332202015-03-11 15:04:43 -0700193 import time
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800194 import json
cameron@onlab.us059c2562015-04-02 14:12:51 -0700195 import string
andrew@onlab.us10332202015-03-11 15:04:43 -0700196 import csv
jenkins3af0cd82015-03-24 10:27:16 -0700197 import numpy
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700198 import os.path
199
200 global currentNeighbors
201 neighbors = []
202
203 try:
204 currentNeighbors
205 except:
206 currentNeighbors = "0"
207 neighbors = ['0']
208 else:
209 if currentNeighbors == "r": #reset
210 currentNeighbors = "a"
211 neighbors = ['0']
212 else:
Jon Hall4ba53f02015-07-29 13:07:41 -0700213 currentNeighbors = "r"
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700214 neighbors = ['a']
215
216 if clusterCount == 1:
217 currentNeighbors = "r"
andrew@onlab.us10332202015-03-11 15:04:43 -0700218
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800219 main.log.info("Cluster Count = " + str(clusterCount))
220
andrew@onlab.us10332202015-03-11 15:04:43 -0700221 intentsRate = main.params['METRICS']['intents_rate']
222 intentsWithdrawn = main.params[ 'METRICS' ][ 'intents_withdrawn' ]
223 intentsFailed = main.params[ 'METRICS' ][ 'intents_failed' ]
224 testDuration = main.params[ 'TEST' ][ 'duration' ]
225 logInterval = main.params[ 'TEST' ][ 'log_interval' ]
226 debug = main.params[ 'debugMode' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -0700227 numKeys = main.params[ 'TEST' ][ 'numKeys' ]
228 cyclePeriod = main.params[ 'TEST' ][ 'cyclePeriod' ]
Jon Hall4ba53f02015-07-29 13:07:41 -0700229 #neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",")
andrew@onlab.us10332202015-03-11 15:04:43 -0700230 metricList = [intentsRate, intentsWithdrawn, intentsFailed]
jenkins3af0cd82015-03-24 10:27:16 -0700231
Jon Hall4ba53f02015-07-29 13:07:41 -0700232 for n in range(0, len(neighbors)):
233 if neighbors[n] == 'a':
cameron@onlab.us059c2562015-04-02 14:12:51 -0700234 neighbors[n] = str(clusterCount -1)
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700235 if int(clusterCount) == 1:
236 neighbors = neighbors.pop()
Jon Hall4ba53f02015-07-29 13:07:41 -0700237
cameron@onlab.us059c2562015-04-02 14:12:51 -0700238 for n in neighbors:
Jon Hall4ba53f02015-07-29 13:07:41 -0700239 main.log.info("Run with " + n + " neighbors")
cameron@onlab.us059c2562015-04-02 14:12:51 -0700240 time.sleep(5)
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700241 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numKeys " + numKeys )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700242 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700243 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numNeighbors " + n )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700244 main.ONOSbench.handle.expect(":~")
245 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller cyclePeriod " + cyclePeriod )
246 main.ONOSbench.handle.expect(":~")
jenkins3af0cd82015-03-24 10:27:16 -0700247
cameron@onlab.us059c2562015-04-02 14:12:51 -0700248 cmd = "onos $OC1 intent-perf-start"
249 main.ONOSbench.handle.sendline(cmd)
250 main.ONOSbench.handle.expect(":~")
251 main.log.info("Starting ONOS (all nodes) intent-perf from $OC1" )
jenkins3af0cd82015-03-24 10:27:16 -0700252
cameron@onlab.us059c2562015-04-02 14:12:51 -0700253 main.log.info( "Starting test loop for " + str(testDuration) + " seconds...\n" )
254 stop = time.time() + float( testDuration )
jenkins3af0cd82015-03-24 10:27:16 -0700255
cameron@onlab.us059c2562015-04-02 14:12:51 -0700256 while time.time() < stop:
257 time.sleep( float( logInterval ) )
258 groupResult = []
259 for node in range (1, clusterCount + 1):
260 groupResult.append(0)
jenkins3af0cd82015-03-24 10:27:16 -0700261
GlennRCb0773152015-09-13 21:02:11 -0700262 cmd = " onos-ssh $OC" + str(node) + """ cat /opt/onos/log/karaf.log | grep "Throughput:" | tail -1 """
cameron@onlab.us059c2562015-04-02 14:12:51 -0700263 main.log.info("COMMAND: " + str(cmd))
264
265 x = 0
266 while True:
267 main.ONOSbench.handle.sendline(cmd)
jenkins50379942015-06-19 13:36:43 -0700268 time.sleep(6)
cameron@onlab.us059c2562015-04-02 14:12:51 -0700269 main.ONOSbench.handle.expect(":~")
270 raw = main.ONOSbench.handle.before
271 if "OVERALL=" in raw:
272 break
273 x += 1
274 if x > 10:
275 main.log.error("Expected output not being recieved... continuing")
276 break
277 time.sleep(2)
278
279 raw = raw.splitlines()
280 splitResults = []
281 for line in raw:
282 splitResults.extend(line.split(" "))
283
284 myResult = "--"
285 for field in splitResults:
286 if "OVERALL" in field:
287 myResult = field
288
289 if myResult == "--":
290 main.log.error("Parsing/Pexpect error\n" + str(splitResults))
291
292 myResult = myResult.replace(";", "")
293 myResult = myResult.replace("OVERALL=","")
294 myResult = float(myResult)
295 groupResult[len(groupResult) -1] = myResult
296
297 main.log.info("Node " + str(node) + " overall rate: " + str(myResult))
298
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700299 clusterTotal = str(numpy.sum(groupResult))
cameron@onlab.us059c2562015-04-02 14:12:51 -0700300 main.log.report("Results from this round of polling: " + str(groupResult))
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700301 main.log.report("Cluster Total: " + clusterTotal + "\n")
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700302
cameron@onlab.us059c2562015-04-02 14:12:51 -0700303 cmd = "onos $OC1 intent-perf-stop"
304 main.ONOSbench.handle.sendline(cmd)
305 main.ONOSbench.handle.expect(":~")
306 main.log.info("Stopping intentperf" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700307
GlennRC47736b82015-09-16 14:27:51 -0700308 with open("/tmp/IntentEventTPDB", "a") as resultsDB:
309 for node in groupResult:
310 resultString = "'" + commit + "',"
311 resultString += "'1gig',"
312 resultString += str(clusterCount) + ","
313 resultString += "'baremetal" + str(int(groupResult.index(node)) + 1) + "',"
314 resultString += n + ","
315 resultString += str(node) + ","
316 resultString += str(0) + "\n" #no stddev
317 resultsDB.write(resultString)
Jon Hall4ba53f02015-07-29 13:07:41 -0700318
319 resultsDB.close()
320
321 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700322