blob: 210134fcc9ad4fe0c976f891b43143271a384491 [file] [log] [blame]
Jon Hallca319892017-06-15 15:25:22 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2017 Open Networking Foundation ( ONF )
Jon Hallca319892017-06-15 15:25:22 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jon Hallca319892017-06-15 15:25:22 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jon Hallca319892017-06-15 15:25:22 -070021class Cluster():
22
23 def __str__( self ):
24 return self.name
Jon Hall4173b242017-09-12 17:04:38 -070025
Jon Hallca319892017-06-15 15:25:22 -070026 def __repr__( self ):
Jon Hallca319892017-06-15 15:25:22 -070027 controllers = []
Devin Lim142b5342017-07-20 15:22:39 -070028 runningNodes = []
Jon Hallca319892017-06-15 15:25:22 -070029 for ctrl in self.controllers:
30 controllers.append( str( ctrl ) )
31 return "%s[%s]" % ( self.name, ", ".join( controllers ) )
32
Jon Hallca319892017-06-15 15:25:22 -070033 def __init__( self, ctrlList=[], name="Cluster" ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070034 """
Devin Lim142b5342017-07-20 15:22:39 -070035 controllers : All the nodes
36 runningNodes : Node that are specifically running from the test.
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070037 ie ) When the test is testing different number of nodes on each
Devin Lim142b5342017-07-20 15:22:39 -070038 run.
39 numCtrls : number of runningNodes
40 maxCtrls : number of controllers
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070041 """
Jon Hallca319892017-06-15 15:25:22 -070042 self.controllers = ctrlList
Devin Lim142b5342017-07-20 15:22:39 -070043 self.runningNodes = ctrlList
44 self.numCtrls = len( self.runningNodes )
45 self.maxCtrls = len( self.controllers )
Jon Hallca319892017-06-15 15:25:22 -070046 self.name = str( name )
47 self.iterator = iter( self.active() )
48
Devin Lim142b5342017-07-20 15:22:39 -070049 def getIps( self, activeOnly=False, allNode=False ):
50 """
51 Description:
52 get the list of the ip. Default to get ips of running nodes.
53 Required:
54 * activeOnly - True for getting ips of active nodes
55 * allNode - True for getting ips of all nodes
56 Returns:
57 Retruns ips of active nodes if activeOnly is True.
58 Returns ips of all nodes if allNode is True.
59 """
Jon Hallca319892017-06-15 15:25:22 -070060 ips = []
Devin Lim142b5342017-07-20 15:22:39 -070061 if allNode:
Jon Hallca319892017-06-15 15:25:22 -070062 nodeList = self.controllers
Devin Lim142b5342017-07-20 15:22:39 -070063 else:
64 if activeOnly:
65 nodeList = self.active()
66 else:
67 nodeList = self.runningNodes
68
Jon Hallca319892017-06-15 15:25:22 -070069 for ctrl in nodeList:
70 ips.append( ctrl.ipAddress )
Devin Lim142b5342017-07-20 15:22:39 -070071
Jon Hallca319892017-06-15 15:25:22 -070072 return ips
73
Devin Lim142b5342017-07-20 15:22:39 -070074 def resetActive( self ):
Jon Hallca319892017-06-15 15:25:22 -070075 """
Devin Lim142b5342017-07-20 15:22:39 -070076 Description:
77 reset the activeness of the runningNodes to be false
78 Required:
79 Returns:
Jon Hallca319892017-06-15 15:25:22 -070080 """
Devin Lim142b5342017-07-20 15:22:39 -070081 for ctrl in self.runningNodes:
82 ctrl.active = False
83
84 def getRunningPos( self ):
85 """
86 Description:
87 get the position of the active running nodes.
88 Required:
89 Returns:
90 Retruns the list of the position of the active
91 running nodes.
92 """
93 return [ ctrl.pos for ctrl in self.runningNodes
94 if ctrl.active ]
95
96 def setRunningNode( self, numCtrls ):
97 """
98 Description:
99 Set running nodes of n number of nodes.
100 It will create new list of runningNodes.
101 If numCtrls is a list, it will add the nodes of the
102 list.
103 Required:
104 * numCtrls - number of nodes to be set.
105 Returns:
106 """
107 self.runningNodes = []
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700108 for i in numCtrls if isinstance( numCtrls, list ) else range( numCtrls ):
Devin Lim142b5342017-07-20 15:22:39 -0700109 self.runningNodes.append( self.controllers[ i ] )
110 self.numCtrls = len( numCtrls ) if isinstance( numCtrls, list ) else numCtrls
111
112 def active( self, node=None ):
113 """
114 Description:
115 get the list/controller of the active controller.
116 Required:
117 * node - position of the node to get from the active controller.
118 Returns:
119 Return a list of active controllers in the cluster if node is None
120 if not, it will return the nth controller.
121 """
122 result = [ ctrl for ctrl in self.runningNodes
Jon Hallca319892017-06-15 15:25:22 -0700123 if ctrl.active ]
Devin Lim142b5342017-07-20 15:22:39 -0700124 return result if node is None else result[ node % len( result ) ]
Jon Hallca319892017-06-15 15:25:22 -0700125
126 def next( self ):
127 """
128 An iterator for the cluster's controllers that
129 resets when there are no more elements.
130
131 Returns the next controller in the cluster
132 """
133 try:
134 return self.iterator.next()
135 except StopIteration:
136 self.reset()
137 try:
138 return self.iterator.next()
139 except StopIteration:
140 raise RuntimeError( "There are no active nodes in the cluster" )
141
142 def reset( self ):
143 """
144 Resets the cluster iterator.
145
146 This should be used whenever a node's active state is changed
147 and is also used internally when the iterator has been exhausted.
148 """
149 self.iterator = iter( self.active() )
150
Devin Lime9f0ccf2017-08-11 17:25:12 -0700151 def createCell( self, cellName, Mininet, useSSH, ips, installMax=False ):
Jon Hallca319892017-06-15 15:25:22 -0700152 """
Devin Lim142b5342017-07-20 15:22:39 -0700153 Description:
154 create a new cell
155 Required:
156 * cellName - The name of the cell.
157 * Mininet - a mininet driver that will be used.
158 * useSSH - True for using ssh when creating a cell
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700159 * ips - ip( s ) of the node( s ).
Devin Lim142b5342017-07-20 15:22:39 -0700160 Returns:
161 """
Devin Lime9f0ccf2017-08-11 17:25:12 -0700162 self.command( "createCellFile",
163 args=[ main.ONOSbench.ip_address,
Devin Lim40a19092017-08-15 14:54:22 -0700164 cellName,
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700165 Mininet if isinstance( Mininet, str ) else
Devin Lim40a19092017-08-15 14:54:22 -0700166 Mininet.ip_address,
167 main.apps,
168 ips,
169 main.ONOScell.karafUser,
170 useSSH ],
Devin Lime9f0ccf2017-08-11 17:25:12 -0700171 specificDriver=1,
172 getFrom=0 if installMax else 1 )
Devin Lim40a19092017-08-15 14:54:22 -0700173
Devin Lim142b5342017-07-20 15:22:39 -0700174 def uninstall( self, uninstallMax ):
175 """
176 Description:
177 uninstalling onos
178 Required:
179 * uninstallMax - True for uninstalling max number of nodes
180 False for uninstalling the current running nodes.
181 Returns:
182 Returns main.TRUE if it successfully uninstalled.
183 """
Devin Lim40a19092017-08-15 14:54:22 -0700184 result = main.TRUE
185 uninstallResult = self.command( "onosUninstall",
Jon Hall4173b242017-09-12 17:04:38 -0700186 kwargs={ "nodeIp": "ipAddress" },
Devin Lim40a19092017-08-15 14:54:22 -0700187 specificDriver=1,
188 getFrom=0 if uninstallMax else 1,
189 funcFromCtrl=True )
190 for uninstallR in uninstallResult:
191 result = result and uninstallR
192 return result
Devin Lim142b5342017-07-20 15:22:39 -0700193
Devin Lime9f0ccf2017-08-11 17:25:12 -0700194 def applyCell( self, cellName, installMax=False ):
Devin Lim142b5342017-07-20 15:22:39 -0700195 """
196 Description:
197 apply the cell with cellName. It will also verify the
198 cell.
199 Required:
200 * cellName - The name of the cell.
201 Returns:
202 Returns main.TRUE if it successfully set and verify cell.
203 """
Devin Lime9f0ccf2017-08-11 17:25:12 -0700204 setCellResult = self.command( "setCell",
Jon Hall4173b242017-09-12 17:04:38 -0700205 args=[ cellName ],
206 specificDriver=1,
207 getFrom=0 if installMax else 1 )
Devin Lime9f0ccf2017-08-11 17:25:12 -0700208 verifyResult = self.command( "verifyCell",
Jon Hall4173b242017-09-12 17:04:38 -0700209 specificDriver=1,
210 getFrom=0 if installMax else 1 )
Devin Lime9f0ccf2017-08-11 17:25:12 -0700211 result = main.TRUE
212 for i in range( len( setCellResult ) ):
213 result = result and setCellResult[ i ] and verifyResult[ i ]
214 return result
Devin Lim142b5342017-07-20 15:22:39 -0700215
216 def checkService( self ):
217 """
218 Description:
219 Checking if the onos service is up. If not, it will
220 start the onos service manually.
221 Required:
222 Returns:
223 Returns main.TRUE if it successfully checked
224 """
225 stopResult = main.TRUE
226 startResult = main.TRUE
227 onosIsUp = main.TRUE
Devin Lim40a19092017-08-15 14:54:22 -0700228 onosUp = self.command( "isup",
229 args=[ "ipAddress" ],
230 specificDriver=1,
231 getFrom=1,
232 funcFromCtrl=True )
233 for i in range( len( onosUp ) ):
234 ctrl = self.controllers[ i ]
235 onosIsUp = onosIsUp and onosUp[ i ]
236 if onosUp[ i ] == main.TRUE:
237 main.log.report( ctrl.name + " is up and ready" )
Devin Lim142b5342017-07-20 15:22:39 -0700238 else:
Devin Lim40a19092017-08-15 14:54:22 -0700239 main.log.report( ctrl.name + " may not be up, stop and " +
Devin Lim142b5342017-07-20 15:22:39 -0700240 "start ONOS again " )
241 stopResult = stopResult and main.ONOSbench.onosStop( ctrl.ipAddress )
242 startResult = startResult and main.ONOSbench.onosStart( ctrl.ipAddress )
243 if not startResult or stopResult:
Devin Lim40a19092017-08-15 14:54:22 -0700244 main.log.report( ctrl.name + " did not start correctly." )
Devin Lim142b5342017-07-20 15:22:39 -0700245 return onosIsUp and stopResult and startResult
246
247 def kill( self, killMax, stopOnos ):
248 """
249 Description:
250 killing the onos. It will either kill the current runningnodes or
251 max number of the nodes.
252 Required:
253 * killRemoveMax - The boolean that will decide either to kill
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700254 only running nodes ( False ) or max number of nodes ( True ).
Devin Lim142b5342017-07-20 15:22:39 -0700255 * stopOnos - If wish to stop onos before killing it. True for
256 enable stop , False for disable stop.
257 Returns:
258 Returns main.TRUE if successfully killing it.
Jon Hallca319892017-06-15 15:25:22 -0700259 """
260 result = main.TRUE
Devin Lim40a19092017-08-15 14:54:22 -0700261 killResult = self.command( "onosKill",
262 args=[ "ipAddress" ],
263 specificDriver=1,
264 getFrom=0 if killMax else 1,
265 funcFromCtrl=True )
Jon Hall4173b242017-09-12 17:04:38 -0700266 for i in range( len( killResult ) ):
Devin Lim40a19092017-08-15 14:54:22 -0700267 result = result and killResult[ i ]
268 self.controllers[ i ].active = False
Devin Lim142b5342017-07-20 15:22:39 -0700269 return result
270
271 def ssh( self ):
272 """
273 Description:
274 set up ssh to the onos
275 Required:
276 Returns:
277 Returns main.TRUE if it successfully setup the ssh to
278 the onos.
279 """
Devin Lim40a19092017-08-15 14:54:22 -0700280 result = main.TRUE
281 sshResult = self.command( "onosSecureSSH",
Jon Hall4173b242017-09-12 17:04:38 -0700282 kwargs={ "node": "ipAddress" },
Devin Lim40a19092017-08-15 14:54:22 -0700283 specificDriver=1,
284 getFrom=1,
285 funcFromCtrl=True )
286 for sshR in sshResult:
287 result = result and sshR
288 return result
Devin Lim142b5342017-07-20 15:22:39 -0700289
Devin Lime9f0ccf2017-08-11 17:25:12 -0700290 def install( self, installMax=True, installParallel=True ):
Devin Lim142b5342017-07-20 15:22:39 -0700291 """
292 Description:
293 Installing onos.
294 Required:
295 * installMax - True for installing max number of nodes
296 False for installing current running nodes only.
297 Returns:
298 Returns main.TRUE if it successfully installed
299 """
300 result = main.TRUE
301 threads = []
302 i = 0
303 for ctrl in self.controllers if installMax else self.runningNodes:
304 options = "-f"
305 if installMax and i >= self.numCtrls:
306 options = "-nf"
Devin Lime9f0ccf2017-08-11 17:25:12 -0700307 if installParallel:
Jon Hall4173b242017-09-12 17:04:38 -0700308 t = main.Thread( target=ctrl.Bench.onosInstall,
309 name="install-" + ctrl.name,
310 kwargs={ "node" : ctrl.ipAddress,
311 "options" : options } )
Devin Lime9f0ccf2017-08-11 17:25:12 -0700312 threads.append( t )
313 t.start()
314 else:
315 result = result and \
Devin Lim142b5342017-07-20 15:22:39 -0700316 main.ONOSbench.onosInstall( node=ctrl.ipAddress, options=options )
Devin Lime9f0ccf2017-08-11 17:25:12 -0700317 if installParallel:
318 for t in threads:
319 t.join()
320 result = result and t.result
Jon Hallca319892017-06-15 15:25:22 -0700321 return result
322
323 def startCLIs( self ):
324 """
Devin Lim142b5342017-07-20 15:22:39 -0700325 Description:
326 starting Onos using onosCli driver
327 Required:
328 Returns:
329 Returns main.TRUE if it successfully started.
Jon Hallca319892017-06-15 15:25:22 -0700330 """
Devin Lim40a19092017-08-15 14:54:22 -0700331 result = main.TRUE
332 cliResults = self.command( "startOnosCli",
333 args=[ "ipAddress" ],
334 specificDriver=2,
335 getFrom=1,
336 funcFromCtrl=True )
Jon Hall4173b242017-09-12 17:04:38 -0700337 for i in range( len( cliResults ) ):
Devin Lim40a19092017-08-15 14:54:22 -0700338 result = result and cliResults[ i ]
339 self.controllers[ i ].active = True
340 return result
Jon Hallca319892017-06-15 15:25:22 -0700341
Devin Lim142b5342017-07-20 15:22:39 -0700342 def printResult( self, results, activeList, logLevel="debug" ):
343 """
344 Description:
345 Print the value of the list.
346 Required:
347 * results - list of the result
348 * activeList - list of the acitve nodes.
349 * logLevel - Type of log level you want it to be printed.
350 Returns:
351 """
352 f = getattr( main.log, logLevel )
Jon Hall4173b242017-09-12 17:04:38 -0700353 for i in range( len( results ) ):
354 f( activeList[ i ].name + "'s result : " + str( results[ i ] ) )
Devin Lim142b5342017-07-20 15:22:39 -0700355
356 def allTrueResultCheck( self, results, activeList ):
357 """
358 Description:
359 check if all the result has main.TRUE.
360 Required:
361 * results - list of the result
362 * activeList - list of the acitve nodes.
363 Returns:
364 Returns True if all == main.TRUE else
365 returns False
366 """
367 self.printResult( results, activeList )
368 return all( result == main.TRUE for result in results )
369
370 def notEmptyResultCheck( self, results, activeList ):
371 """
372 Description:
373 check if all the result has any contents
374 Required:
375 * results - list of the result
376 * activeList - list of the acitve nodes.
377 Returns:
378 Returns True if all the results has
379 something else returns False
380 """
381 self.printResult( results, activeList )
382 return all( result for result in results )
383
384 def identicalResultsCheck( self, results, activeList ):
385 """
386 Description:
387 check if all the results has same output.
388 Required:
389 * results - list of the result
390 * activeList - list of the acitve nodes.
391 Returns:
392 Returns True if all the results has
393 same result else returns False
394 """
395 self.printResult( results, activeList )
396 resultOne = results[ 0 ]
397 return all( resultOne == result for result in results )
398
Devin Lime9f0ccf2017-08-11 17:25:12 -0700399 def command( self, function, args=(), kwargs={}, returnBool=False,
Devin Lim40a19092017-08-15 14:54:22 -0700400 specificDriver=0, contentCheck=False, getFrom=2,
401 funcFromCtrl=False ):
Devin Lim142b5342017-07-20 15:22:39 -0700402 """
403 Description:
404 execute some function of the active nodes.
405 Required:
406 * function - name of the function
407 * args - argument of the function
408 * kwargs - kwargs of the funciton
409 * returnBool - True if wish to check all the result has main.TRUE
410 * specificDriver - specific driver to execute the function. Since
411 some of the function can be used in different drivers, it is important
412 to specify which driver it will be executed from.
413 0 - any type of driver
414 1 - from bench
415 2 - from cli
416 3 - from rest
417 * contentCheck - If this is True, it will check if the result has some
418 contents.
Devin Lime9f0ccf2017-08-11 17:25:12 -0700419 * getFrom - from which nodes
420 2 - active nodes
421 1 - current running nodes
422 0 - all nodes
Devin Lim40a19092017-08-15 14:54:22 -0700423 * funcFromCtrl - specific function of the args/kwargs
424 from each controller from the list of the controllers
Devin Lim142b5342017-07-20 15:22:39 -0700425 Returns:
426 Returns results if not returnBool and not contentCheck
427 Returns checkTruthValue of the result if returnBool
428 Returns resultContent of the result if contentCheck
429 """
Jon Hallca319892017-06-15 15:25:22 -0700430 threads = []
Devin Lim142b5342017-07-20 15:22:39 -0700431 drivers = [ None, "Bench", "CLI", "REST" ]
Devin Lime9f0ccf2017-08-11 17:25:12 -0700432 fromNode = [ self.controllers, self.runningNodes, self.active() ]
Jon Hallca319892017-06-15 15:25:22 -0700433 results = []
Devin Lime9f0ccf2017-08-11 17:25:12 -0700434 for ctrl in fromNode[ getFrom ]:
Devin Lim142b5342017-07-20 15:22:39 -0700435 try:
Devin Lim40a19092017-08-15 14:54:22 -0700436 funcArgs = []
437 funcKwargs = {}
Devin Lim142b5342017-07-20 15:22:39 -0700438 f = getattr( ( ctrl if not specificDriver else
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700439 getattr( ctrl, drivers[ specificDriver ] ) ), function )
Devin Lim40a19092017-08-15 14:54:22 -0700440 if funcFromCtrl:
441 if args:
442 for i in range( len( args ) ):
443 funcArgs.append( getattr( ctrl, args[ i ] ) )
444 if kwargs:
445 for k in kwargs:
Jon Hall4173b242017-09-12 17:04:38 -0700446 funcKwargs.update( { k: getattr( ctrl, kwargs[ k ] ) } )
Devin Lim142b5342017-07-20 15:22:39 -0700447 except AttributeError:
448 main.log.error( "Function " + function + " not found. Exiting the Test." )
Devin Lim44075962017-08-11 10:56:37 -0700449 main.cleanAndExit()
Jon Hallca319892017-06-15 15:25:22 -0700450 t = main.Thread( target=f,
451 name=function + "-" + ctrl.name,
Devin Lim40a19092017-08-15 14:54:22 -0700452 args=funcArgs if funcFromCtrl else args,
453 kwargs=funcKwargs if funcFromCtrl else kwargs )
Jon Hallca319892017-06-15 15:25:22 -0700454 threads.append( t )
455 t.start()
456
457 for t in threads:
458 t.join()
459 results.append( t.result )
Devin Lim142b5342017-07-20 15:22:39 -0700460 if returnBool:
Devin Lime9f0ccf2017-08-11 17:25:12 -0700461 return self.allTrueResultCheck( results, fromNode[ getFrom ] )
Devin Lim142b5342017-07-20 15:22:39 -0700462 elif contentCheck:
Devin Lime9f0ccf2017-08-11 17:25:12 -0700463 return self.notEmptyResultCheck( results, fromNode[ getFrom ] )
Jon Hall4173b242017-09-12 17:04:38 -0700464 return results
465
466 def checkPartitionSize( self, segmentSize='64', units='M', multiplier='3' ):
467 # max segment size in bytes: 1024 * 1024 * 64
468 # multiplier is somewhat arbitrary, but the idea is the logs would have
469 # been compacted before this many segments are written
470
471 maxSize = float( segmentSize ) * float( multiplier )
472 ret = True
473 for n in self.runningNodes:
474 ret = ret and n.server.folderSize( "/opt/onos/apache-karaf-*/data/partitions/*/*.log",
475 size=maxSize, unit=units, ignoreRoot=False )
476 return ret