blob: 44178b2212a79b0135c54826cd628a68d0d443bc [file] [log] [blame]
cameron@onlab.us059c2562015-04-02 14:12:51 -07001# ScaleOutTemplate
andrew@onlab.us2ae3a112015-02-02 11:24:32 -08002#
3# CASE1 starts number of nodes specified in param file
4#
5# cameron@onlab.us
6
andrew@onlab.us10332202015-03-11 15:04:43 -07007import sys
cameron@onlab.us059c2562015-04-02 14:12:51 -07008import os.path
andrew@onlab.us10332202015-03-11 15:04:43 -07009
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080010
11class IntentEventTP:
12
13 def __init__( self ):
14 self.default = ''
15
cameron@onlab.us059c2562015-04-02 14:12:51 -070016 def CASE1( self, main ):
17
18 import time
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070019 import os.path
20
cameron@onlab.us059c2562015-04-02 14:12:51 -070021 global init
22 try:
23 if type(init) is not bool:
24 init = False
25 except NameError:
26 init = False
27
andrew@onlab.us10332202015-03-11 15:04:43 -070028 #Load values from params file
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080029 checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
30 gitPull = main.params[ 'GIT' ][ 'autopull' ]
31 cellName = main.params[ 'ENV' ][ 'cellName' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070032 Apps = main.params[ 'ENV' ][ 'cellApps' ]
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080033 BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
34 BENCHUser = main.params[ 'BENCH' ][ 'user' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070035 MN1Ip = main.params[ 'MN' ][ 'ip1' ]
andrew@onlab.us10332202015-03-11 15:04:43 -070036 maxNodes = int(main.params[ 'availableNodes' ])
andrew@onlab.us10332202015-03-11 15:04:43 -070037 skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -070038 cellName = main.params[ 'ENV' ][ 'cellName' ]
jenkins3af0cd82015-03-24 10:27:16 -070039 numSwitches = (main.params[ 'TEST' ][ 'numSwitches' ]).split(",")
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -070040 flowRuleBU = main.params[ 'TEST' ][ 'flowRuleBUEnabled' ]
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070041 onBaremetal = main.params['isOnBaremetal']
42 homeDir = os.path.expanduser('~')
andrew@onlab.us2ae3a112015-02-02 11:24:32 -080043
cameron@onlab.us412e9562015-05-13 16:04:34 -070044 main.exceptions = [0]*11
45 main.warnings = [0]*11
46 main.errors = [0]*11
47
cameron@onlab.us059c2562015-04-02 14:12:51 -070048 # -- INIT SECTION, ONLY RUNS ONCE -- #
49 if init == False:
50 init = True
51 global clusterCount #number of nodes running
52 global ONOSIp #list of ONOS IP addresses
53 global scale
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070054 global commit
cameron@onlab.usc10e22c2015-05-13 13:07:28 -070055
cameron@onlab.us059c2562015-04-02 14:12:51 -070056 clusterCount = 0
57 ONOSIp = [ 0 ]
58 scale = (main.params[ 'SCALE' ]).split(",")
59 clusterCount = int(scale[0])
60
61 #Populate ONOSIp with ips from params
jenkins15b2b132015-06-23 14:04:09 -070062 ONOSIp = [0]
63 ONOSIp.extend(main.ONOSbench.getOnosIps())
jenkins15b2b132015-06-23 14:04:09 -070064
cameron@onlab.us059c2562015-04-02 14:12:51 -070065 #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
66 if skipMvn != "yes":
67 mvnResult = main.ONOSbench.cleanInstall()
68
69 #git
70 main.step( "Git checkout and pull " + checkoutBranch )
71 if gitPull == 'on':
72 checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
73 pullResult = main.ONOSbench.gitPull()
74
75 else:
76 checkoutResult = main.TRUE
77 pullResult = main.TRUE
78 main.log.info( "Skipped git checkout and pull" )
andrew@onlab.us10332202015-03-11 15:04:43 -070079
cameron@onlab.usc80a8c82015-04-15 14:57:37 -070080 commit = main.ONOSbench.getVersion()
81 commit = (commit.split(" "))[1]
82
83 resultsDB = open("IntentEventTPDB", "w+")
84 resultsDB.close()
85
cameron@onlab.us059c2562015-04-02 14:12:51 -070086 # -- END OF INIT SECTION --#
87
88 clusterCount = int(scale[0])
89 scale.remove(scale[0])
jenkins5a771992015-06-24 13:53:27 -070090
91 MN1Ip = ONOSIp[len(ONOSIp) -1]
92 BENCHIp = ONOSIp[len(ONOSIp) -2]
93
cameron@onlab.us059c2562015-04-02 14:12:51 -070094 #kill off all onos processes
andrew@onlab.us10332202015-03-11 15:04:43 -070095 main.log.step("Safety check, killing all ONOS processes")
96 main.log.step("before initiating enviornment setup")
97 for node in range(1, maxNodes + 1):
andrew@onlab.us10332202015-03-11 15:04:43 -070098 main.ONOSbench.onosDie(ONOSIp[node])
jenkins8f8f4f32015-06-24 14:05:04 -070099
100 MN1Ip = ONOSIp[len(ONOSIp) -1]
101 BENCHIp = ONOSIp[len(ONOSIp) -2]
102
andrew@onlab.us10332202015-03-11 15:04:43 -0700103 #Uninstall everywhere
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800104 main.log.step( "Cleaning Enviornment..." )
andrew@onlab.us10332202015-03-11 15:04:43 -0700105 for i in range(1, maxNodes + 1):
106 main.log.info(" Uninstalling ONOS " + str(i) )
107 main.ONOSbench.onosUninstall( ONOSIp[i] )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700108
109 #construct the cell file
110 main.log.info("Creating cell file")
111 cellIp = []
112 for node in range (1, clusterCount + 1):
113 cellIp.append(ONOSIp[node])
jenkins5a771992015-06-24 13:53:27 -0700114
cameron@onlab.us059c2562015-04-02 14:12:51 -0700115 main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), *cellIp)
116
117 main.step( "Set Cell" )
118 main.ONOSbench.setCell(cellName)
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800119
jenkins3af0cd82015-03-24 10:27:16 -0700120 myDistribution = []
121 for node in range (1, clusterCount + 1):
122 myDistribution.append(numSwitches[node-1])
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800123
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700124 #main.ONOSbench.createLinkGraphFile( BENCHIp,cellIp,myDistribution)
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700125
126 if onBaremetal == "True":
127 filename = "/onos/tools/package/bin/onos-service"
128 serviceConfig = open(homeDir + filename, 'w+')
129 serviceConfig.write("#!/bin/bash\n ")
130 serviceConfig.write("#------------------------------------- \n ")
131 serviceConfig.write("# Starts ONOS Apache Karaf container\n ")
132 serviceConfig.write("#------------------------------------- \n ")
133 serviceConfig.write("#export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}\n ")
134 serviceConfig.write("""export JAVA_OPTS="${JAVA_OPTS:--Xms8G -Xmx8G}" \n """)
135 serviceConfig.write("")
136 serviceConfig.write("ONOS_HOME=/opt/onos \n ")
137 serviceConfig.write("")
138 serviceConfig.write("[ -d $ONOS_HOME ] && cd $ONOS_HOME || ONOS_HOME=$(dirname $0)/..\n")
139 serviceConfig.write("""${ONOS_HOME}/apache-karaf-$KARAF_VERSION/bin/karaf "$@" \n """)
140 serviceConfig.close()
141 main.log.info("Set /onos/tools/package/bin/onos-service with 8G Xms/Xmx Options.")
142
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800143 main.step( "Creating ONOS package" )
144 packageResult = main.ONOSbench.onosPackage()
145
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800146 main.step( "verify cells" )
147 verifyCellResult = main.ONOSbench.verifyCell()
cameron@onlab.us059c2562015-04-02 14:12:51 -0700148
149 main.log.report( "Initializeing " + str( clusterCount ) + " node cluster." )
andrew@onlab.us10332202015-03-11 15:04:43 -0700150 for node in range(1, clusterCount + 1):
cameron@onlab.us059c2562015-04-02 14:12:51 -0700151 main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
andrew@onlab.us10332202015-03-11 15:04:43 -0700152 main.ONOSbench.onosInstall( ONOSIp[node])
cameron@onlab.us059c2562015-04-02 14:12:51 -0700153
cameron@onlab.us1201bc42015-04-01 16:30:05 -0700154 for node in range(1, clusterCount + 1):
andrew@onlab.us10332202015-03-11 15:04:43 -0700155 for i in range( 2 ):
156 isup = main.ONOSbench.isup( ONOSIp[node] )
157 if isup:
158 main.log.info("ONOS " + str(node) + " is up\n")
159 break
160 if not isup:
161 main.log.report( "ONOS " + str(node) + " didn't start!" )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700162 main.log.info("Startup sequence complete")
163
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700164 time.sleep(20)
165
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700166
jenkins50379942015-06-19 13:36:43 -0700167 for i in range(5):
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700168 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount """ + str(clusterCount*10) + """ " """)
169 main.ONOSbench.handle.expect(":~")
170 main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
171 main.ONOSbench.handle.expect(":~")
172 if ("value=" + str(clusterCount*10)) in main.ONOSbench.handle.before:
173 main.log.info("Device count set")
174 main.log.info("before" + main.ONOSbench.handle.before)
175 break
176 time.sleep(10)
177 main.log.info("cfg set failure, retrying")
178 main.log.info("before" + main.ONOSbench.handle.before)
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700179
jenkins50379942015-06-19 13:36:43 -0700180 for i in range(5):
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700181 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
182 main.ONOSbench.handle.expect(":~")
183 main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
184 main.ONOSbench.handle.expect(":~")
185 if ("value=linear") in main.ONOSbench.handle.before:
186 main.log.info("Device count set")
187 main.log.info("before" + main.ONOSbench.handle.before)
188 break
189 time.sleep(10)
190 main.log.info("cfg set failure, retrying")
191 main.log.info("before" + main.ONOSbench.handle.before)
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700192
193 main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBU + """" """)
194 main.ONOSbench.handle.expect(":~")
195 main.ONOSbench.handle.sendline("""onos $OC1 "cfg get" """)
196 main.ONOSbench.handle.expect(":~")
197 main.log.info(main.ONOSbench.handle.before)
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700198
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700199 time.sleep(10)
200 main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
201 main.ONOSbench.handle.expect(":~")
202 print main.ONOSbench.handle.before
203 time.sleep(10)
204 main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
205 main.ONOSbench.handle.expect(":~")
206 print main.ONOSbench.handle.before
207
andrew@onlab.us10332202015-03-11 15:04:43 -0700208 lastOutput = "--"
209 origin = time.time()
210 clockStarted = False
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700211 while True:
andrew@onlab.us10332202015-03-11 15:04:43 -0700212 main.ONOSbench.handle.sendline("onos $OC1 summary")
213 main.ONOSbench.handle.expect(":~")
cameron@onlab.usf33b45c2015-05-06 13:54:46 -0700214 main.log.info("before" + main.ONOSbench.handle.before)
andrew@onlab.us10332202015-03-11 15:04:43 -0700215 clusterCheck = ((main.ONOSbench.handle.before).splitlines())[3]
216 print("\nBefore: " + str(clusterCheck))
cameron@onlab.us059c2562015-04-02 14:12:51 -0700217 if ("SCC(s)=1,") in clusterCheck:
andrew@onlab.us10332202015-03-11 15:04:43 -0700218 break
219 if clusterCheck != lastOutput:
220 sameOutput = False
221 elif clusterCheck == lastOutput:
222 if clockStarted == False:
223 start = time.time()
224 clockStarted = True
cameron@onlab.us059c2562015-04-02 14:12:51 -0700225 if time.time() > (start + 10):
andrew@onlab.us10332202015-03-11 15:04:43 -0700226 main.log.error("TIMEOUT EXCEEDED: Clusters have not converged, continuing anyway...")
227 break
228 lastOutput = clusterCheck
229 time.sleep(5)
jenkins50379942015-06-19 13:36:43 -0700230 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
cameron@onlab.us059c2562015-04-02 14:12:51 -0700231 def CASE2( self, main ):
andrew@onlab.us10332202015-03-11 15:04:43 -0700232 import time
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800233 import json
cameron@onlab.us059c2562015-04-02 14:12:51 -0700234 import string
andrew@onlab.us10332202015-03-11 15:04:43 -0700235 import csv
jenkins3af0cd82015-03-24 10:27:16 -0700236 import numpy
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700237 import os.path
238
239 global currentNeighbors
240 neighbors = []
241
242 try:
243 currentNeighbors
244 except:
245 currentNeighbors = "0"
246 neighbors = ['0']
247 else:
248 if currentNeighbors == "r": #reset
249 currentNeighbors = "a"
250 neighbors = ['0']
251 else:
252 currentNeighbors = "r"
253 neighbors = ['a']
254
255 if clusterCount == 1:
256 currentNeighbors = "r"
andrew@onlab.us10332202015-03-11 15:04:43 -0700257
andrew@onlab.us2ae3a112015-02-02 11:24:32 -0800258 main.log.info("Cluster Count = " + str(clusterCount))
259
andrew@onlab.us10332202015-03-11 15:04:43 -0700260 intentsRate = main.params['METRICS']['intents_rate']
261 intentsWithdrawn = main.params[ 'METRICS' ][ 'intents_withdrawn' ]
262 intentsFailed = main.params[ 'METRICS' ][ 'intents_failed' ]
263 testDuration = main.params[ 'TEST' ][ 'duration' ]
264 logInterval = main.params[ 'TEST' ][ 'log_interval' ]
265 debug = main.params[ 'debugMode' ]
cameron@onlab.us059c2562015-04-02 14:12:51 -0700266 numKeys = main.params[ 'TEST' ][ 'numKeys' ]
267 cyclePeriod = main.params[ 'TEST' ][ 'cyclePeriod' ]
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700268 #neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",")
andrew@onlab.us10332202015-03-11 15:04:43 -0700269 metricList = [intentsRate, intentsWithdrawn, intentsFailed]
jenkins3af0cd82015-03-24 10:27:16 -0700270
cameron@onlab.us059c2562015-04-02 14:12:51 -0700271 for n in range(0, len(neighbors)):
272 if neighbors[n] == 'a':
273 neighbors[n] = str(clusterCount -1)
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700274 if int(clusterCount) == 1:
275 neighbors = neighbors.pop()
276
cameron@onlab.us059c2562015-04-02 14:12:51 -0700277 for n in neighbors:
278 main.log.info("Run with " + n + " neighbors")
279 time.sleep(5)
cameron@onlab.uscd4e8a22015-05-11 10:58:43 -0700280 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numKeys " + numKeys )
cameron@onlab.us059c2562015-04-02 14:12:51 -0700281 main.ONOSbench.handle.expect(":~")
282 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numNeighbors " + n )
283 main.ONOSbench.handle.expect(":~")
284 main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller cyclePeriod " + cyclePeriod )
285 main.ONOSbench.handle.expect(":~")
jenkins3af0cd82015-03-24 10:27:16 -0700286
cameron@onlab.us059c2562015-04-02 14:12:51 -0700287 cmd = "onos $OC1 intent-perf-start"
288 main.ONOSbench.handle.sendline(cmd)
289 main.ONOSbench.handle.expect(":~")
290 main.log.info("Starting ONOS (all nodes) intent-perf from $OC1" )
jenkins3af0cd82015-03-24 10:27:16 -0700291
cameron@onlab.us059c2562015-04-02 14:12:51 -0700292 main.log.info( "Starting test loop for " + str(testDuration) + " seconds...\n" )
293 stop = time.time() + float( testDuration )
jenkins3af0cd82015-03-24 10:27:16 -0700294
cameron@onlab.us059c2562015-04-02 14:12:51 -0700295 while time.time() < stop:
296 time.sleep( float( logInterval ) )
297 groupResult = []
298 for node in range (1, clusterCount + 1):
299 groupResult.append(0)
jenkins3af0cd82015-03-24 10:27:16 -0700300
cameron@onlab.us059c2562015-04-02 14:12:51 -0700301 cmd = " onos-ssh $OC" + str(node) + """ cat /opt/onos/log/karaf.log | grep "SNAPSHOT | Throughput" | tail -1 """
302 main.log.info("COMMAND: " + str(cmd))
303
304 x = 0
305 while True:
306 main.ONOSbench.handle.sendline(cmd)
jenkins50379942015-06-19 13:36:43 -0700307 time.sleep(6)
cameron@onlab.us059c2562015-04-02 14:12:51 -0700308 main.ONOSbench.handle.expect(":~")
309 raw = main.ONOSbench.handle.before
310 if "OVERALL=" in raw:
311 break
312 x += 1
313 if x > 10:
314 main.log.error("Expected output not being recieved... continuing")
315 break
316 time.sleep(2)
317
318 raw = raw.splitlines()
319 splitResults = []
320 for line in raw:
321 splitResults.extend(line.split(" "))
322
323 myResult = "--"
324 for field in splitResults:
325 if "OVERALL" in field:
326 myResult = field
327
328 if myResult == "--":
329 main.log.error("Parsing/Pexpect error\n" + str(splitResults))
330
331 myResult = myResult.replace(";", "")
332 myResult = myResult.replace("OVERALL=","")
333 myResult = float(myResult)
334 groupResult[len(groupResult) -1] = myResult
335
336 main.log.info("Node " + str(node) + " overall rate: " + str(myResult))
337
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700338 clusterTotal = str(numpy.sum(groupResult))
cameron@onlab.us059c2562015-04-02 14:12:51 -0700339 main.log.report("Results from this round of polling: " + str(groupResult))
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700340 main.log.report("Cluster Total: " + clusterTotal + "\n")
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700341
cameron@onlab.us059c2562015-04-02 14:12:51 -0700342 cmd = "onos $OC1 intent-perf-stop"
343 main.ONOSbench.handle.sendline(cmd)
344 main.ONOSbench.handle.expect(":~")
345 main.log.info("Stopping intentperf" )
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700346
347 resultsDB = open("IntentEventTPDB", "a")
348 for node in groupResult:
349
350 resultString = "'" + commit + "',"
351 resultString += "'1gig',"
352 resultString += str(clusterCount) + ","
353 resultString += "'baremetal" + str(int(groupResult.index(node)) + 1) + "',"
354 resultString += n + ","
355 resultString += str(node) + ","
356 resultString += str(0) + "\n" #no stddev
357 resultsDB.write(resultString)
358
359 resultsDB.close()
cameron@onlab.usc10e22c2015-05-13 13:07:28 -0700360
jenkins50379942015-06-19 13:36:43 -0700361 main.ONOSbench.logReport(ONOSIp[1], ["ERROR", "WARNING", "EXCEPT"])
cameron@onlab.usc80a8c82015-04-15 14:57:37 -0700362