blob: fb3f7db3124b06f26260bc826e29a7336512c4f8 [file] [log] [blame]
cameron@onlab.us059c2562015-04-02 14:12:51 -07001# ScaleOutTemplate
andrew@onlab.us2ae3a112015-02-02 11:24:32 -08002#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
andrew@onlab.us10332202015-03-11 15:04:43 -07007import sys
cameron@onlab.us059c2562015-04-02 14:12:51 -07008import os.path
cameron@onlab.usaaecfd72015-07-08 12:27:26 -07009import time
andrew@onlab.us10332202015-03-11 15:04:43 -070010
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080011
suibin7f2c9cd2015-07-08 17:34:59 -070012class SCPFintentEventTp:
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080013
14 def __init__( self ):
15 self.default = ''
16
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070017 def CASE1( self, main ):
18 import sys
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070019 import os.path
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070020 import time
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070021
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070022 global init
23 try:
24 if type(init) is not bool:
25 init = Fals
26 except NameError:
27 init = False
28
andrew@onlab.us10332202015-03-11 15:04:43 -070029 #Load values from params file
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080030 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
31 gitPull = main.params[ 'GIT' ][ 'autopull' ]
32 cellName = main.params[ 'ENV' ][ 'cellName' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070033 Apps = main.params[ 'ENV' ][ 'cellApps' ]
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080034 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
35 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070036 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070037 maxNodes = int(main.params[ 'max' ])
Jon Hall4ba53f02015-07-29 13:07:41 -070038 main.maxNodes = maxNodes
andrew@onlab.us10332202015-03-11 15:04:43 -070039 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
cameron@onlab.usaaecfd72015-07-08 12:27:26 -070040 cellName = main.params[ 'ENV' ][ 'cellName' ]
jenkins3af0cd82015-03-24 10:27:16 -070041 numSwitches = (main.params[ 'TEST' ][ 'numSwitches' ]).split(",")
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -070042 flowRuleBU = main.params[ 'TEST' ][ 'flowRuleBUEnabled' ]
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070043 homeDir = os.path.expanduser('~')
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080044
cameron@onlab.us412e9562015-05-13 16:04:34 -070045 main.exceptions = [0]*11
46 main.warnings = [0]*11
47 main.errors = [0]*11
48
Jon Hall4ba53f02015-07-29 13:07:41 -070049 # -- INIT SECTION, ONLY RUNS ONCE -- #
50 if init == False:
cameron@onlab.us059c2562015-04-02 14:12:51 -070051 init = True
52 global clusterCount #number of nodes running
53 global ONOSIp #list of ONOS IP addresses
Jon Hall4ba53f02015-07-29 13:07:41 -070054 global scale
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070055 global commit
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070056
cameron@onlab.us059c2562015-04-02 14:12:51 -070057 clusterCount = 0
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070058 ONOSIp = main.ONOSbench.getOnosIps()
59 print ONOSIp
60 print main.ONOSbench.onosIps.values()
61
Jon Hall4ba53f02015-07-29 13:07:41 -070062 scale = (main.params[ 'SCALE' ]).split(",")
cameron@onlab.us059c2562015-04-02 14:12:51 -070063 clusterCount = int(scale[0])
64
Jon Hall4ba53f02015-07-29 13:07:41 -070065 #Populate ONOSIp with ips from params
jenkins15b2b132015-06-23 14:04:09 -070066 ONOSIp.extend(main.ONOSbench.getOnosIps())
jenkins15b2b132015-06-23 14:04:09 -070067
cameron@onlab.us059c2562015-04-02 14:12:51 -070068 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
69 if skipMvn != "yes":
70 mvnResult = main.ONOSbench.cleanInstall()
71
72 #git
73 main.step( "Git checkout and pull " + checkoutBranch )
74 if gitPull == 'on':
75 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
76 pullResult = main.ONOSbench.gitPull()
77
78 else:
79 checkoutResult = main.TRUE
80 pullResult = main.TRUE
81 main.log.info( "Skipped git checkout and pull" )
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070082
Jon Hall4ba53f02015-07-29 13:07:41 -070083 main.log.step("Grabbing commit number")
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070084 commit = main.ONOSbench.getVersion()
85 commit = (commit.split(" "))[1]
Jon Hall4ba53f02015-07-29 13:07:41 -070086
87 main.log.step("Creating results file")
suibin584c0702015-07-14 15:57:27 -070088 resultsDB = open("/tmp/IntentEventTPDB", "w+")
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070089 resultsDB.close()
90
cameron@onlab.us059c2562015-04-02 14:12:51 -070091 # -- END OF INIT SECTION --#
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070092
Jon Hall4ba53f02015-07-29 13:07:41 -070093 main.log.step("Adjusting scale")
94 print str(scale)
cameron@onlab.usc5bf8192015-07-13 13:36:05 -070095 print str(ONOSIp)
cameron@onlab.us059c2562015-04-02 14:12:51 -070096 clusterCount = int(scale[0])
Jon Hall4ba53f02015-07-29 13:07:41 -070097 scale.remove(scale[0])
98
jenkins5a771992015-06-24 13:53:27 -070099 MN1Ip = ONOSIp[len(ONOSIp) -1]
100 BENCHIp = ONOSIp[len(ONOSIp) -2]
Jon Hall4ba53f02015-07-29 13:07:41 -0700101
102 #kill off all onos processes
andrew@onlab.us10332202015-03-11 15:04:43 -0700103 main.log.step("Safety check, killing all ONOS processes")
104 main.log.step("before initiating enviornment setup")
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700105 for node in range(maxNodes):
andrew@onlab.us10332202015-03-11 15:04:43 -0700106 main.ONOSbench.onosDie(ONOSIp[node])
Jon Hall4ba53f02015-07-29 13:07:41 -0700107
jenkins8f8f4f32015-06-24 14:05:04 -0700108 MN1Ip = ONOSIp[len(ONOSIp) -1]
109 BENCHIp = ONOSIp[len(ONOSIp) -2]
110
andrew@onlab.us10332202015-03-11 15:04:43 -0700111 #Uninstall everywhere
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800112 main.log.step( "Cleaning Enviornment..." )
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700113 for i in range(maxNodes):
andrew@onlab.us10332202015-03-11 15:04:43 -0700114 main.log.info(" Uninstalling ONOS " + str(i) )
115 main.ONOSbench.onosUninstall( ONOSIp[i] )
suibine1a104d2015-07-08 15:58:29 -0700116 main.log.info("Sleep 10 second for uninstall to settle...")
117 time.sleep(10)
118 main.ONOSbench.handle.sendline(" ")
119 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700120
cameron@onlab.us059c2562015-04-02 14:12:51 -0700121 #construct the cell file
122 main.log.info("Creating cell file")
123 cellIp = []
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700124 for node in range (clusterCount):
cameron@onlab.us059c2562015-04-02 14:12:51 -0700125 cellIp.append(ONOSIp[node])
Jon Hall4ba53f02015-07-29 13:07:41 -0700126
cameron@onlab.usc5bf8192015-07-13 13:36:05 -0700127 main.ONOSbench.createCellFile("localhost",cellName,MN1Ip,str(Apps), cellIp)
cameron@onlab.us059c2562015-04-02 14:12:51 -0700128
129 main.step( "Set Cell" )
130 main.ONOSbench.setCell(cellName)
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800131
jenkins3af0cd82015-03-24 10:27:16 -0700132 myDistribution = []
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700133 for node in range (clusterCount):
134 myDistribution.append(numSwitches[node])
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800135
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800136 main.step( "Creating ONOS package" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700137 packageResult = main.ONOSbench.onosPackage()
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800138
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800139 main.step( "verify cells" )
140 verifyCellResult = main.ONOSbench.verifyCell()
Jon Hall4ba53f02015-07-29 13:07:41 -0700141
cameron@onlab.us059c2562015-04-02 14:12:51 -0700142 main.log.report( "Initializeing " + str( clusterCount ) + " node cluster." )
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700143 for node in range(clusterCount):
cameron@onlab.us059c2562015-04-02 14:12:51 -0700144 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
andrew@onlab.us10332202015-03-11 15:04:43 -0700145 main.ONOSbench.onosInstall( ONOSIp[node])
cameron@onlab.us059c2562015-04-02 14:12:51 -0700146
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700147 for node in range(clusterCount):
andrew@onlab.us10332202015-03-11 15:04:43 -0700148 for i in range( 2 ):
149 isup = main.ONOSbench.isup( ONOSIp[node] )
150 if isup:
151 main.log.info("ONOS " + str(node) + " is up\n")
152 break
153 if not isup:
154 main.log.report( "ONOS " + str(node) + " didn't start!" )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700155 main.log.info("Startup sequence complete")
156
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700157 time.sleep(20)
158
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700159 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.store.flow.impl.NewDistributedFlowRuleStore", "backupEnabled false")
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700160
Jon Hall4ba53f02015-07-29 13:07:41 -0700161 devices = int(clusterCount)*10
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700162
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700163 main.log.step("Setting up null provider")
164 for i in range(3):
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700165 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "deviceCount " + str(devices))
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700166 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "topoShape linear")
167 main.ONOSbench.onosCfgSet( ONOSIp[0], "org.onosproject.provider.nil.NullProviders", "enabled true")
168 time.sleep(5)
169
170 main.ONOSbench.handle.sendline("onos $OC1 summary")
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700171 main.ONOSbench.handle.expect(":~")
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700172
173 before = main.ONOSbench.handle.before
cameron@onlab.useae03dc2015-07-29 11:51:28 -0700174 if ("devices=" + str(devices)) in before:
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700175 break
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700176
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700177 main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
178 main.ONOSbench.handle.expect(":~")
179 print main.ONOSbench.handle.before
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700180
andrew@onlab.us10332202015-03-11 15:04:43 -0700181 lastOutput = "--"
182 origin = time.time()
183 clockStarted = False
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700184 while True:
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700185
186 main.ONOSbench.handle.sendline("")
187 main.ONOSbench.handle.expect(":~")
188
andrew@onlab.us10332202015-03-11 15:04:43 -0700189 main.ONOSbench.handle.sendline("onos $OC1 summary")
190 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700191
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700192 main.log.info("before" + main.ONOSbench.handle.before)
Jon Hallf57a5ef2015-07-07 17:56:16 -0700193 clusterCheck = main.ONOSbench.handle.before
andrew@onlab.us10332202015-03-11 15:04:43 -0700194 print("\nBefore: " + str(clusterCheck))
cameron@onlab.us059c2562015-04-02 14:12:51 -0700195 if ("SCC(s)=1,") in clusterCheck:
andrew@onlab.us10332202015-03-11 15:04:43 -0700196 break
197 if clusterCheck != lastOutput:
198 sameOutput = False
199 elif clusterCheck == lastOutput:
200 if clockStarted == False:
201 start = time.time()
202 clockStarted = True
cameron@onlab.usaaecfd72015-07-08 12:27:26 -0700203 if time.time() > (start + 30):
andrew@onlab.us10332202015-03-11 15:04:43 -0700204 main.log.error("TIMEOUT EXCEEDED: Clusters have not converged, continuing anyway...")
205 break
206 lastOutput = clusterCheck
207 time.sleep(5)
jenkins50379942015-06-19 13:36:43 -0700208 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
Jon Hall4ba53f02015-07-29 13:07:41 -0700209 def CASE2( self, main ):
andrew@onlab.us10332202015-03-11 15:04:43 -0700210 import time
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800211 import json
cameron@onlab.us059c2562015-04-02 14:12:51 -0700212 import string
andrew@onlab.us10332202015-03-11 15:04:43 -0700213 import csv
jenkins3af0cd82015-03-24 10:27:16 -0700214 import numpy
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700215 import os.path
216
217 global currentNeighbors
218 neighbors = []
219
220 try:
221 currentNeighbors
222 except:
223 currentNeighbors = "0"
224 neighbors = ['0']
225 else:
226 if currentNeighbors == "r": #reset
227 currentNeighbors = "a"
228 neighbors = ['0']
229 else:
Jon Hall4ba53f02015-07-29 13:07:41 -0700230 currentNeighbors = "r"
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700231 neighbors = ['a']
232
233 if clusterCount == 1:
234 currentNeighbors = "r"
andrew@onlab.us10332202015-03-11 15:04:43 -0700235
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800236 main.log.info("Cluster Count = " + str(clusterCount))
237
andrew@onlab.us10332202015-03-11 15:04:43 -0700238 intentsRate = main.params['METRICS']['intents_rate']
239 intentsWithdrawn = main.params[ 'METRICS' ][ 'intents_withdrawn' ]
240 intentsFailed = main.params[ 'METRICS' ][ 'intents_failed' ]
241 testDuration = main.params[ 'TEST' ][ 'duration' ]
242 logInterval = main.params[ 'TEST' ][ 'log_interval' ]
243 debug = main.params[ 'debugMode' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -0700244 numKeys = main.params[ 'TEST' ][ 'numKeys' ]
245 cyclePeriod = main.params[ 'TEST' ][ 'cyclePeriod' ]
Jon Hall4ba53f02015-07-29 13:07:41 -0700246 #neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",")
andrew@onlab.us10332202015-03-11 15:04:43 -0700247 metricList = [intentsRate, intentsWithdrawn, intentsFailed]
jenkins3af0cd82015-03-24 10:27:16 -0700248
Jon Hall4ba53f02015-07-29 13:07:41 -0700249 for n in range(0, len(neighbors)):
250 if neighbors[n] == 'a':
cameron@onlab.us059c2562015-04-02 14:12:51 -0700251 neighbors[n] = str(clusterCount -1)
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700252 if int(clusterCount) == 1:
253 neighbors = neighbors.pop()
Jon Hall4ba53f02015-07-29 13:07:41 -0700254
cameron@onlab.us059c2562015-04-02 14:12:51 -0700255 for n in neighbors:
Jon Hall4ba53f02015-07-29 13:07:41 -0700256 main.log.info("Run with " + n + " neighbors")
cameron@onlab.us059c2562015-04-02 14:12:51 -0700257 time.sleep(5)
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700258 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numKeys " + numKeys )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700259 main.ONOSbench.handle.expect(":~")
Jon Hall4ba53f02015-07-29 13:07:41 -0700260 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numNeighbors " + n )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700261 main.ONOSbench.handle.expect(":~")
262 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller cyclePeriod " + cyclePeriod )
263 main.ONOSbench.handle.expect(":~")
jenkins3af0cd82015-03-24 10:27:16 -0700264
cameron@onlab.us059c2562015-04-02 14:12:51 -0700265 cmd = "onos $OC1 intent-perf-start"
266 main.ONOSbench.handle.sendline(cmd)
267 main.ONOSbench.handle.expect(":~")
268 main.log.info("Starting ONOS (all nodes) intent-perf from $OC1" )
jenkins3af0cd82015-03-24 10:27:16 -0700269
cameron@onlab.us059c2562015-04-02 14:12:51 -0700270 main.log.info( "Starting test loop for " + str(testDuration) + " seconds...\n" )
271 stop = time.time() + float( testDuration )
jenkins3af0cd82015-03-24 10:27:16 -0700272
cameron@onlab.us059c2562015-04-02 14:12:51 -0700273 while time.time() < stop:
274 time.sleep( float( logInterval ) )
275 groupResult = []
276 for node in range (1, clusterCount + 1):
277 groupResult.append(0)
jenkins3af0cd82015-03-24 10:27:16 -0700278
cameron@onlab.us059c2562015-04-02 14:12:51 -0700279 cmd = " onos-ssh $OC" + str(node) + """ cat /opt/onos/log/karaf.log | grep "SNAPSHOT | Throughput" | tail -1 """
280 main.log.info("COMMAND: " + str(cmd))
281
282 x = 0
283 while True:
284 main.ONOSbench.handle.sendline(cmd)
jenkins50379942015-06-19 13:36:43 -0700285 time.sleep(6)
cameron@onlab.us059c2562015-04-02 14:12:51 -0700286 main.ONOSbench.handle.expect(":~")
287 raw = main.ONOSbench.handle.before
288 if "OVERALL=" in raw:
289 break
290 x += 1
291 if x > 10:
292 main.log.error("Expected output not being recieved... continuing")
293 break
294 time.sleep(2)
295
296 raw = raw.splitlines()
297 splitResults = []
298 for line in raw:
299 splitResults.extend(line.split(" "))
300
301 myResult = "--"
302 for field in splitResults:
303 if "OVERALL" in field:
304 myResult = field
305
306 if myResult == "--":
307 main.log.error("Parsing/Pexpect error\n" + str(splitResults))
308
309 myResult = myResult.replace(";", "")
310 myResult = myResult.replace("OVERALL=","")
311 myResult = float(myResult)
312 groupResult[len(groupResult) -1] = myResult
313
314 main.log.info("Node " + str(node) + " overall rate: " + str(myResult))
315
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700316 clusterTotal = str(numpy.sum(groupResult))
cameron@onlab.us059c2562015-04-02 14:12:51 -0700317 main.log.report("Results from this round of polling: " + str(groupResult))
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700318 main.log.report("Cluster Total: " + clusterTotal + "\n")
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700319
cameron@onlab.us059c2562015-04-02 14:12:51 -0700320 cmd = "onos $OC1 intent-perf-stop"
321 main.ONOSbench.handle.sendline(cmd)
322 main.ONOSbench.handle.expect(":~")
323 main.log.info("Stopping intentperf" )
Jon Hall4ba53f02015-07-29 13:07:41 -0700324
suibine1e58772015-07-15 09:52:59 -0700325 resultsDB = open("/tmp/IntentEventTPDB", "a")
Jon Hall4ba53f02015-07-29 13:07:41 -0700326 for node in groupResult:
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700327
328 resultString = "'" + commit + "',"
329 resultString += "'1gig',"
330 resultString += str(clusterCount) + ","
331 resultString += "'baremetal" + str(int(groupResult.index(node)) + 1) + "',"
332 resultString += n + ","
333 resultString += str(node) + ","
334 resultString += str(0) + "\n" #no stddev
335 resultsDB.write(resultString)
Jon Hall4ba53f02015-07-29 13:07:41 -0700336
337 resultsDB.close()
338
339 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700340