blob: 385d9e6dc31812b1835a7842ba9741a013a89b2a [file] [log] [blame]
cameron@onlab.us059c2562015-04-02 14:12:51 -07001# ScaleOutTemplate
andrew@onlab.us2ae3a112015-02-02 11:24:32 -08002#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
andrew@onlab.us10332202015-03-11 15:04:43 -07007import sys
cameron@onlab.us059c2562015-04-02 14:12:51 -07008import os.path
cameron@onlab.usaaecfd72015-07-08 12:27:26 -07009import time
andrew@onlab.us10332202015-03-11 15:04:43 -070010
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080011
suibin7f2c9cd2015-07-08 17:34:59 -070012class SCPFintentEventTp:
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080013
14 def __init__( self ):
15 self.default = ''
16
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070017 def CASE1( self, main ):
18 import sys
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070019 import os.path
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070020 import time
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070021
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070022 global init
23 try:
24 if type(init) is not bool:
GlennRCf9ef27d2015-09-16 12:08:38 -070025 init = False
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070026 except NameError:
27 init = False
28
andrew@onlab.us10332202015-03-11 15:04:43 -070029 #Load values from params file
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080030 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
31 gitPull = main.params[ 'GIT' ][ 'autopull' ]
32 cellName = main.params[ 'ENV' ][ 'cellName' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070033 Apps = main.params[ 'ENV' ][ 'cellApps' ]
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080034 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
35 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070036 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070037 maxNodes = int(main.params[ 'max' ])
Jon Hall4ba53f02015-07-29 13:07:41 -070038 main.maxNodes = maxNodes
andrew@onlab.us10332202015-03-11 15:04:43 -070039 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070040 cellName = main.params[ 'ENV' ][ 'cellName' ]
jenkins3af0cd82015-03-24 10:27:16 -070041 numSwitches = (main.params[ 'TEST' ][ 'numSwitches' ]).split(",")
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -070042 flowRuleBU = main.params[ 'TEST' ][ 'flowRuleBUEnabled' ]
suibin zhangd1f03802016-02-25 11:48:58 -080043 skipRelRsrc = main.params[ 'TEST'][ 'skipReleaseResourcesOnWithdrawal']
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070044 homeDir = os.path.expanduser('~')
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080045
cameron@onlab.us412e9562015-05-13 16:04:34 -070046 main.exceptions = [0]*11
47 main.warnings = [0]*11
48 main.errors = [0]*11
49
Jon Hall4ba53f02015-07-29 13:07:41 -070050 # -- INIT SECTION, ONLY RUNS ONCE -- #
51 if init == False:
cameron@onlab.us059c2562015-04-02 14:12:51 -070052 init = True
53 global clusterCount #number of nodes running
54 global ONOSIp #list of ONOS IP addresses
Jon Hall4ba53f02015-07-29 13:07:41 -070055 global scale
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070056 global commit
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070057
cameron@onlab.us059c2562015-04-02 14:12:51 -070058 clusterCount = 0
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070059 ONOSIp = main.ONOSbench.getOnosIps()
60 print ONOSIp
61 print main.ONOSbench.onosIps.values()
62
Jon Hall4ba53f02015-07-29 13:07:41 -070063 scale = (main.params[ 'SCALE' ]).split(",")
cameron@onlab.us059c2562015-04-02 14:12:51 -070064 clusterCount = int(scale[0])
65
Jon Hall4ba53f02015-07-29 13:07:41 -070066 #Populate ONOSIp with ips from params
jenkins15b2b132015-06-23 14:04:09 -070067 ONOSIp.extend(main.ONOSbench.getOnosIps())
jenkins15b2b132015-06-23 14:04:09 -070068
cameron@onlab.us059c2562015-04-02 14:12:51 -070069 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
70 if skipMvn != "yes":
71 mvnResult = main.ONOSbench.cleanInstall()
72
73 #git
74 main.step( "Git checkout and pull " + checkoutBranch )
75 if gitPull == 'on':
76 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
77 pullResult = main.ONOSbench.gitPull()
78
79 else:
80 checkoutResult = main.TRUE
81 pullResult = main.TRUE
82 main.log.info( "Skipped git checkout and pull" )
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070083
Jon Hall4ba53f02015-07-29 13:07:41 -070084 main.log.step("Grabbing commit number")
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070085 commit = main.ONOSbench.getVersion()
86 commit = (commit.split(" "))[1]
Jon Hall4ba53f02015-07-29 13:07:41 -070087
88 main.log.step("Creating results file")
suibin584c0702015-07-14 15:57:27 -070089 resultsDB = open("/tmp/IntentEventTPDB", "w+")
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070090 resultsDB.close()
91
cameron@onlab.us059c2562015-04-02 14:12:51 -070092 # -- END OF INIT SECTION --#
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070093
Jon Hall4ba53f02015-07-29 13:07:41 -070094 main.log.step("Adjusting scale")
95 print str(scale)
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070096 print str(ONOSIp)
cameron@onlab.us059c2562015-04-02 14:12:51 -070097 clusterCount = int(scale[0])
Jon Hall4ba53f02015-07-29 13:07:41 -070098 scale.remove(scale[0])
99
jenkins5a771992015-06-24 13:53:27 -0700100 MN1Ip = ONOSIp[len(ONOSIp) -1]
101 BENCHIp = ONOSIp[len(ONOSIp) -2]
Jon Hall4ba53f02015-07-29 13:07:41 -0700102
103 #kill off all onos processes
andrew@onlab.us10332202015-03-11 15:04:43 -0700104 main.log.step("Safety check, killing all ONOS processes")
Jon Hall70b2ff42015-11-17 15:49:44 -0800105 main.log.step("before initiating environment setup")
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700106 for node in range(maxNodes):
andrew@onlab.us10332202015-03-11 15:04:43 -0700107 main.ONOSbench.onosDie(ONOSIp[node])
Jon Hall4ba53f02015-07-29 13:07:41 -0700108
jenkins8f8f4f32015-06-24 14:05:04 -0700109 MN1Ip = ONOSIp[len(ONOSIp) -1]
110 BENCHIp = ONOSIp[len(ONOSIp) -2]
111
andrew@onlab.us10332202015-03-11 15:04:43 -0700112 #Uninstall everywhere
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800113 main.log.step( "Cleaning Enviornment..." )
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700114 for i in range(maxNodes):
andrew@onlab.us10332202015-03-11 15:04:43 -0700115 main.log.info(" Uninstalling ONOS " + str(i) )
116 main.ONOSbench.onosUninstall( ONOSIp[i] )
suibine1a104d2015-07-08 15:58:29 -0700117 main.log.info("Sleep 10 second for uninstall to settle...")
118 time.sleep(10)
119 main.ONOSbench.handle.sendline(" ")
120 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700121
cameron@onlab.us059c2562015-04-02 14:12:51 -0700122 #construct the cell file
123 main.log.info("Creating cell file")
124 cellIp = []
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700125 for node in range (clusterCount):
cameron@onlab.us059c2562015-04-02 14:12:51 -0700126 cellIp.append(ONOSIp[node])
Jon Hall4ba53f02015-07-29 13:07:41 -0700127
cameron@onlab.usc5bf8192015-07-13 13:36:05 -0700128 main.ONOSbench.createCellFile("localhost",cellName,MN1Ip,str(Apps), cellIp)
cameron@onlab.us059c2562015-04-02 14:12:51 -0700129
130 main.step( "Set Cell" )
131 main.ONOSbench.setCell(cellName)
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800132
jenkins3af0cd82015-03-24 10:27:16 -0700133 myDistribution = []
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700134 for node in range (clusterCount):
135 myDistribution.append(numSwitches[node])
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800136
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800137 main.step( "Creating ONOS package" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700138 packageResult = main.ONOSbench.onosPackage()
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800139
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800140 main.step( "verify cells" )
141 verifyCellResult = main.ONOSbench.verifyCell()
Jon Hall4ba53f02015-07-29 13:07:41 -0700142
cameron@onlab.us059c2562015-04-02 14:12:51 -0700143 main.log.report( "Initializeing " + str( clusterCount ) + " node cluster." )
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700144 for node in range(clusterCount):
cameron@onlab.us059c2562015-04-02 14:12:51 -0700145 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
andrew@onlab.us10332202015-03-11 15:04:43 -0700146 main.ONOSbench.onosInstall( ONOSIp[node])
cameron@onlab.us059c2562015-04-02 14:12:51 -0700147
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700148 for node in range(clusterCount):
andrew@onlab.us10332202015-03-11 15:04:43 -0700149 for i in range( 2 ):
150 isup = main.ONOSbench.isup( ONOSIp[node] )
151 if isup:
152 main.log.info("ONOS " + str(node) + " is up\n")
153 break
154 if not isup:
155 main.log.report( "ONOS " + str(node) + " didn't start!" )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700156 main.log.info("Startup sequence complete")
157
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700158 time.sleep(20)
159
suibin zhang0b9346b2015-09-03 12:22:52 -0700160 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.store.flow.impl.NewDistributedFlowRuleStore", "backupEnabled " + str(flowRuleBU))
suibin zhangd1f03802016-02-25 11:48:58 -0800161 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.net.intent.impl.IntentManager", "skipReleaseResourcesOnWithdrawal " + skipRelRsrc)
Jon Hall4ba53f02015-07-29 13:07:41 -0700162 devices = int(clusterCount)*10
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700163
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700164 main.log.step("Setting up null provider")
165 for i in range(3):
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700166 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "deviceCount " + str(devices))
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700167 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "topoShape linear")
168 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "enabled true")
169 time.sleep(5)
170
171 main.ONOSbench.handle.sendline("onos $OC1 summary")
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700172 main.ONOSbench.handle.expect(":~")
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700173
174 before = main.ONOSbench.handle.before
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700175 if ("devices=" + str(devices)) in before:
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700176 break
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700177
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700178 main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
179 main.ONOSbench.handle.expect(":~")
180 print main.ONOSbench.handle.before
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700181
GlennRCf9ef27d2015-09-16 12:08:38 -0700182 for i in range(3):
183 passed = main.ONOSbench.verifySummary( ONOSIp[0] )
184 if passed:
185 main.log.info("Clusters have converged")
andrew@onlab.us10332202015-03-11 15:04:43 -0700186 break
GlennRCf9ef27d2015-09-16 12:08:38 -0700187 else:
188 main.log.error("Clusters have not converged, retying...")
189 time.sleep(3)
190
jenkins50379942015-06-19 13:36:43 -0700191 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
GlennRCf9ef27d2015-09-16 12:08:38 -0700192
Jon Hall4ba53f02015-07-29 13:07:41 -0700193 def CASE2( self, main ):
andrew@onlab.us10332202015-03-11 15:04:43 -0700194 import time
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800195 import json
cameron@onlab.us059c2562015-04-02 14:12:51 -0700196 import string
andrew@onlab.us10332202015-03-11 15:04:43 -0700197 import csv
jenkins3af0cd82015-03-24 10:27:16 -0700198 import numpy
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700199 import os.path
200
201 global currentNeighbors
202 neighbors = []
203
204 try:
205 currentNeighbors
206 except:
207 currentNeighbors = "0"
208 neighbors = ['0']
209 else:
210 if currentNeighbors == "r": #reset
211 currentNeighbors = "a"
212 neighbors = ['0']
213 else:
Jon Hall4ba53f02015-07-29 13:07:41 -0700214 currentNeighbors = "r"
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700215 neighbors = ['a']
216
217 if clusterCount == 1:
218 currentNeighbors = "r"
andrew@onlab.us10332202015-03-11 15:04:43 -0700219
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800220 main.log.info("Cluster Count = " + str(clusterCount))
221
andrew@onlab.us10332202015-03-11 15:04:43 -0700222 intentsRate = main.params['METRICS']['intents_rate']
223 intentsWithdrawn = main.params[ 'METRICS' ][ 'intents_withdrawn' ]
224 intentsFailed = main.params[ 'METRICS' ][ 'intents_failed' ]
225 testDuration = main.params[ 'TEST' ][ 'duration' ]
226 logInterval = main.params[ 'TEST' ][ 'log_interval' ]
227 debug = main.params[ 'debugMode' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -0700228 numKeys = main.params[ 'TEST' ][ 'numKeys' ]
229 cyclePeriod = main.params[ 'TEST' ][ 'cyclePeriod' ]
Jon Hall4ba53f02015-07-29 13:07:41 -0700230 #neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",")
andrew@onlab.us10332202015-03-11 15:04:43 -0700231 metricList = [intentsRate, intentsWithdrawn, intentsFailed]
jenkins3af0cd82015-03-24 10:27:16 -0700232
Jon Hall4ba53f02015-07-29 13:07:41 -0700233 for n in range(0, len(neighbors)):
234 if neighbors[n] == 'a':
cameron@onlab.us059c2562015-04-02 14:12:51 -0700235 neighbors[n] = str(clusterCount -1)
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700236 if int(clusterCount) == 1:
237 neighbors = neighbors.pop()
Jon Hall4ba53f02015-07-29 13:07:41 -0700238
cameron@onlab.us059c2562015-04-02 14:12:51 -0700239 for n in neighbors:
Jon Hall4ba53f02015-07-29 13:07:41 -0700240 main.log.info("Run with " + n + " neighbors")
cameron@onlab.us059c2562015-04-02 14:12:51 -0700241 time.sleep(5)
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700242 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numKeys " + numKeys )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700243 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700244 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numNeighbors " + n )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700245 main.ONOSbench.handle.expect(":~")
246 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller cyclePeriod " + cyclePeriod )
247 main.ONOSbench.handle.expect(":~")
jenkins3af0cd82015-03-24 10:27:16 -0700248
cameron@onlab.us059c2562015-04-02 14:12:51 -0700249 cmd = "onos $OC1 intent-perf-start"
250 main.ONOSbench.handle.sendline(cmd)
251 main.ONOSbench.handle.expect(":~")
252 main.log.info("Starting ONOS (all nodes) intent-perf from $OC1" )
jenkins3af0cd82015-03-24 10:27:16 -0700253
cameron@onlab.us059c2562015-04-02 14:12:51 -0700254 main.log.info( "Starting test loop for " + str(testDuration) + " seconds...\n" )
255 stop = time.time() + float( testDuration )
jenkins3af0cd82015-03-24 10:27:16 -0700256
cameron@onlab.us059c2562015-04-02 14:12:51 -0700257 while time.time() < stop:
258 time.sleep( float( logInterval ) )
259 groupResult = []
260 for node in range (1, clusterCount + 1):
261 groupResult.append(0)
jenkins3af0cd82015-03-24 10:27:16 -0700262
GlennRCb0773152015-09-13 21:02:11 -0700263 cmd = " onos-ssh $OC" + str(node) + """ cat /opt/onos/log/karaf.log | grep "Throughput:" | tail -1 """
cameron@onlab.us059c2562015-04-02 14:12:51 -0700264 main.log.info("COMMAND: " + str(cmd))
265
266 x = 0
267 while True:
268 main.ONOSbench.handle.sendline(cmd)
jenkins50379942015-06-19 13:36:43 -0700269 time.sleep(6)
cameron@onlab.us059c2562015-04-02 14:12:51 -0700270 main.ONOSbench.handle.expect(":~")
271 raw = main.ONOSbench.handle.before
272 if "OVERALL=" in raw:
273 break
274 x += 1
275 if x > 10:
276 main.log.error("Expected output not being recieved... continuing")
277 break
278 time.sleep(2)
279
280 raw = raw.splitlines()
281 splitResults = []
282 for line in raw:
283 splitResults.extend(line.split(" "))
284
285 myResult = "--"
286 for field in splitResults:
287 if "OVERALL" in field:
288 myResult = field
289
290 if myResult == "--":
291 main.log.error("Parsing/Pexpect error\n" + str(splitResults))
292
293 myResult = myResult.replace(";", "")
294 myResult = myResult.replace("OVERALL=","")
295 myResult = float(myResult)
296 groupResult[len(groupResult) -1] = myResult
297
298 main.log.info("Node " + str(node) + " overall rate: " + str(myResult))
299
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700300 clusterTotal = str(numpy.sum(groupResult))
cameron@onlab.us059c2562015-04-02 14:12:51 -0700301 main.log.report("Results from this round of polling: " + str(groupResult))
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700302 main.log.report("Cluster Total: " + clusterTotal + "\n")
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700303
cameron@onlab.us059c2562015-04-02 14:12:51 -0700304 cmd = "onos $OC1 intent-perf-stop"
305 main.ONOSbench.handle.sendline(cmd)
306 main.ONOSbench.handle.expect(":~")
307 main.log.info("Stopping intentperf" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700308
GlennRC47736b82015-09-16 14:27:51 -0700309 with open("/tmp/IntentEventTPDB", "a") as resultsDB:
310 for node in groupResult:
311 resultString = "'" + commit + "',"
312 resultString += "'1gig',"
313 resultString += str(clusterCount) + ","
314 resultString += "'baremetal" + str(int(groupResult.index(node)) + 1) + "',"
315 resultString += n + ","
316 resultString += str(node) + ","
317 resultString += str(0) + "\n" #no stddev
318 resultsDB.write(resultString)
Jon Hall4ba53f02015-07-29 13:07:41 -0700319
320 resultsDB.close()
321
322 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700323