blob: a67bad540928f198605a005188a59a00eed02871 [file] [log] [blame]
Jon Hallca319892017-06-15 15:25:22 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2017 Open Networking Foundation ( ONF )
Jon Hallca319892017-06-15 15:25:22 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jon Hallca319892017-06-15 15:25:22 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Devin Lim3ebd5e72017-11-14 10:38:00 -080021import json
Jon Hallca319892017-06-15 15:25:22 -070022class Cluster():
23
24 def __str__( self ):
25 return self.name
Jon Hall4173b242017-09-12 17:04:38 -070026
Jon Hallca319892017-06-15 15:25:22 -070027 def __repr__( self ):
Jon Hallca319892017-06-15 15:25:22 -070028 controllers = []
Devin Lim142b5342017-07-20 15:22:39 -070029 runningNodes = []
Jon Hallca319892017-06-15 15:25:22 -070030 for ctrl in self.controllers:
31 controllers.append( str( ctrl ) )
32 return "%s[%s]" % ( self.name, ", ".join( controllers ) )
33
Jon Hallca319892017-06-15 15:25:22 -070034 def __init__( self, ctrlList=[], name="Cluster" ):
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070035 """
Devin Lim142b5342017-07-20 15:22:39 -070036 controllers : All the nodes
37 runningNodes : Node that are specifically running from the test.
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070038 ie ) When the test is testing different number of nodes on each
Devin Lim142b5342017-07-20 15:22:39 -070039 run.
40 numCtrls : number of runningNodes
41 maxCtrls : number of controllers
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070042 """
Jon Hallca319892017-06-15 15:25:22 -070043 self.controllers = ctrlList
Devin Lim142b5342017-07-20 15:22:39 -070044 self.runningNodes = ctrlList
45 self.numCtrls = len( self.runningNodes )
46 self.maxCtrls = len( self.controllers )
Jon Hallca319892017-06-15 15:25:22 -070047 self.name = str( name )
48 self.iterator = iter( self.active() )
49
Devin Lim142b5342017-07-20 15:22:39 -070050 def getIps( self, activeOnly=False, allNode=False ):
51 """
52 Description:
53 get the list of the ip. Default to get ips of running nodes.
54 Required:
55 * activeOnly - True for getting ips of active nodes
56 * allNode - True for getting ips of all nodes
57 Returns:
58 Retruns ips of active nodes if activeOnly is True.
59 Returns ips of all nodes if allNode is True.
60 """
Jon Hallca319892017-06-15 15:25:22 -070061 ips = []
Devin Lim142b5342017-07-20 15:22:39 -070062 if allNode:
Jon Hallca319892017-06-15 15:25:22 -070063 nodeList = self.controllers
Devin Lim142b5342017-07-20 15:22:39 -070064 else:
65 if activeOnly:
66 nodeList = self.active()
67 else:
68 nodeList = self.runningNodes
69
Jon Hallca319892017-06-15 15:25:22 -070070 for ctrl in nodeList:
71 ips.append( ctrl.ipAddress )
Devin Lim142b5342017-07-20 15:22:39 -070072
Jon Hallca319892017-06-15 15:25:22 -070073 return ips
74
Jon Halla1e8e512018-05-11 13:30:57 -070075 def clearActive( self ):
Jon Hallca319892017-06-15 15:25:22 -070076 """
Devin Lim142b5342017-07-20 15:22:39 -070077 Description:
Jon Halla1e8e512018-05-11 13:30:57 -070078 Sets the activeness of each cluster node to be False
Jon Hallca319892017-06-15 15:25:22 -070079 """
Jon Hallab611372018-02-21 15:26:05 -080080 for ctrl in self.controllers:
Devin Lim142b5342017-07-20 15:22:39 -070081 ctrl.active = False
82
83 def getRunningPos( self ):
84 """
85 Description:
86 get the position of the active running nodes.
87 Required:
88 Returns:
89 Retruns the list of the position of the active
90 running nodes.
91 """
92 return [ ctrl.pos for ctrl in self.runningNodes
93 if ctrl.active ]
94
95 def setRunningNode( self, numCtrls ):
96 """
97 Description:
98 Set running nodes of n number of nodes.
99 It will create new list of runningNodes.
100 If numCtrls is a list, it will add the nodes of the
101 list.
102 Required:
103 * numCtrls - number of nodes to be set.
104 Returns:
105 """
106 self.runningNodes = []
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700107 for i in numCtrls if isinstance( numCtrls, list ) else range( numCtrls ):
Devin Lim142b5342017-07-20 15:22:39 -0700108 self.runningNodes.append( self.controllers[ i ] )
109 self.numCtrls = len( numCtrls ) if isinstance( numCtrls, list ) else numCtrls
110
111 def active( self, node=None ):
112 """
113 Description:
114 get the list/controller of the active controller.
115 Required:
116 * node - position of the node to get from the active controller.
117 Returns:
118 Return a list of active controllers in the cluster if node is None
119 if not, it will return the nth controller.
120 """
121 result = [ ctrl for ctrl in self.runningNodes
Jon Hallca319892017-06-15 15:25:22 -0700122 if ctrl.active ]
Devin Lim142b5342017-07-20 15:22:39 -0700123 return result if node is None else result[ node % len( result ) ]
Jon Hallca319892017-06-15 15:25:22 -0700124
125 def next( self ):
126 """
Jon Halla1e8e512018-05-11 13:30:57 -0700127 An iterator for the cluster's active controllers that
Jon Hallca319892017-06-15 15:25:22 -0700128 resets when there are no more elements.
129
130 Returns the next controller in the cluster
131 """
132 try:
Jon Halla1e8e512018-05-11 13:30:57 -0700133 node = self.iterator.next()
134 assert node.active
135 return node
136 except ( StopIteration, AssertionError ):
Jon Hallca319892017-06-15 15:25:22 -0700137 self.reset()
138 try:
139 return self.iterator.next()
140 except StopIteration:
141 raise RuntimeError( "There are no active nodes in the cluster" )
142
143 def reset( self ):
144 """
145 Resets the cluster iterator.
146
147 This should be used whenever a node's active state is changed
148 and is also used internally when the iterator has been exhausted.
149 """
150 self.iterator = iter( self.active() )
151
You Wanga0f6ff62018-01-11 15:46:30 -0800152 def createCell( self, cellName, cellApps, mininetIp, useSSH, ips, installMax=False ):
Jon Hallca319892017-06-15 15:25:22 -0700153 """
Devin Lim142b5342017-07-20 15:22:39 -0700154 Description:
155 create a new cell
156 Required:
157 * cellName - The name of the cell.
You Wang09b596b2018-01-10 10:42:38 -0800158 * cellApps - The ONOS apps string.
You Wanga0f6ff62018-01-11 15:46:30 -0800159 * mininetIp - Mininet IP address.
Devin Lim142b5342017-07-20 15:22:39 -0700160 * useSSH - True for using ssh when creating a cell
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700161 * ips - ip( s ) of the node( s ).
Devin Lim142b5342017-07-20 15:22:39 -0700162 Returns:
163 """
You Wang09b596b2018-01-10 10:42:38 -0800164 try:
165 apps = main.apps
166 except ( NameError, AttributeError ):
167 apps = cellApps
Devin Lime9f0ccf2017-08-11 17:25:12 -0700168 self.command( "createCellFile",
169 args=[ main.ONOSbench.ip_address,
Devin Lim40a19092017-08-15 14:54:22 -0700170 cellName,
You Wanga0f6ff62018-01-11 15:46:30 -0800171 mininetIp,
You Wang09b596b2018-01-10 10:42:38 -0800172 apps,
Devin Lim40a19092017-08-15 14:54:22 -0700173 ips,
174 main.ONOScell.karafUser,
175 useSSH ],
Devin Lime9f0ccf2017-08-11 17:25:12 -0700176 specificDriver=1,
177 getFrom=0 if installMax else 1 )
Devin Lim40a19092017-08-15 14:54:22 -0700178
Devin Lim142b5342017-07-20 15:22:39 -0700179 def uninstall( self, uninstallMax ):
180 """
181 Description:
182 uninstalling onos
183 Required:
184 * uninstallMax - True for uninstalling max number of nodes
185 False for uninstalling the current running nodes.
186 Returns:
187 Returns main.TRUE if it successfully uninstalled.
188 """
Devin Lim40a19092017-08-15 14:54:22 -0700189 result = main.TRUE
190 uninstallResult = self.command( "onosUninstall",
Jon Hall4173b242017-09-12 17:04:38 -0700191 kwargs={ "nodeIp": "ipAddress" },
Devin Lim40a19092017-08-15 14:54:22 -0700192 specificDriver=1,
193 getFrom=0 if uninstallMax else 1,
194 funcFromCtrl=True )
195 for uninstallR in uninstallResult:
196 result = result and uninstallR
197 return result
Devin Lim142b5342017-07-20 15:22:39 -0700198
Devin Lime9f0ccf2017-08-11 17:25:12 -0700199 def applyCell( self, cellName, installMax=False ):
Devin Lim142b5342017-07-20 15:22:39 -0700200 """
201 Description:
202 apply the cell with cellName. It will also verify the
203 cell.
204 Required:
205 * cellName - The name of the cell.
206 Returns:
207 Returns main.TRUE if it successfully set and verify cell.
208 """
Devin Lime9f0ccf2017-08-11 17:25:12 -0700209 setCellResult = self.command( "setCell",
Jon Hall4173b242017-09-12 17:04:38 -0700210 args=[ cellName ],
211 specificDriver=1,
212 getFrom=0 if installMax else 1 )
Devin Lime9f0ccf2017-08-11 17:25:12 -0700213 verifyResult = self.command( "verifyCell",
Jon Hall4173b242017-09-12 17:04:38 -0700214 specificDriver=1,
215 getFrom=0 if installMax else 1 )
Devin Lime9f0ccf2017-08-11 17:25:12 -0700216 result = main.TRUE
217 for i in range( len( setCellResult ) ):
218 result = result and setCellResult[ i ] and verifyResult[ i ]
219 return result
Devin Lim142b5342017-07-20 15:22:39 -0700220
221 def checkService( self ):
222 """
223 Description:
224 Checking if the onos service is up. If not, it will
225 start the onos service manually.
226 Required:
227 Returns:
228 Returns main.TRUE if it successfully checked
229 """
230 stopResult = main.TRUE
231 startResult = main.TRUE
232 onosIsUp = main.TRUE
Devin Lim40a19092017-08-15 14:54:22 -0700233 onosUp = self.command( "isup",
234 args=[ "ipAddress" ],
235 specificDriver=1,
236 getFrom=1,
237 funcFromCtrl=True )
238 for i in range( len( onosUp ) ):
239 ctrl = self.controllers[ i ]
240 onosIsUp = onosIsUp and onosUp[ i ]
241 if onosUp[ i ] == main.TRUE:
Jon Hall9655dcb2018-02-02 11:19:06 -0800242 main.log.info( ctrl.name + " is up and ready" )
Devin Lim142b5342017-07-20 15:22:39 -0700243 else:
Jon Hall9655dcb2018-02-02 11:19:06 -0800244 main.log.warn( ctrl.name + " may not be up." )
245 return onosIsUp
Devin Lim142b5342017-07-20 15:22:39 -0700246
247 def kill( self, killMax, stopOnos ):
248 """
249 Description:
250 killing the onos. It will either kill the current runningnodes or
251 max number of the nodes.
252 Required:
253 * killRemoveMax - The boolean that will decide either to kill
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700254 only running nodes ( False ) or max number of nodes ( True ).
Devin Lim142b5342017-07-20 15:22:39 -0700255 * stopOnos - If wish to stop onos before killing it. True for
256 enable stop , False for disable stop.
257 Returns:
258 Returns main.TRUE if successfully killing it.
Jon Hallca319892017-06-15 15:25:22 -0700259 """
260 result = main.TRUE
Devin Lim40a19092017-08-15 14:54:22 -0700261 killResult = self.command( "onosKill",
262 args=[ "ipAddress" ],
263 specificDriver=1,
264 getFrom=0 if killMax else 1,
265 funcFromCtrl=True )
Jon Hall4173b242017-09-12 17:04:38 -0700266 for i in range( len( killResult ) ):
Devin Lim40a19092017-08-15 14:54:22 -0700267 result = result and killResult[ i ]
268 self.controllers[ i ].active = False
Devin Lim142b5342017-07-20 15:22:39 -0700269 return result
270
271 def ssh( self ):
272 """
273 Description:
274 set up ssh to the onos
275 Required:
276 Returns:
277 Returns main.TRUE if it successfully setup the ssh to
278 the onos.
279 """
Devin Lim40a19092017-08-15 14:54:22 -0700280 result = main.TRUE
281 sshResult = self.command( "onosSecureSSH",
Jon Hall4173b242017-09-12 17:04:38 -0700282 kwargs={ "node": "ipAddress" },
Devin Lim40a19092017-08-15 14:54:22 -0700283 specificDriver=1,
284 getFrom=1,
285 funcFromCtrl=True )
286 for sshR in sshResult:
287 result = result and sshR
288 return result
Devin Lim142b5342017-07-20 15:22:39 -0700289
Devin Lime9f0ccf2017-08-11 17:25:12 -0700290 def install( self, installMax=True, installParallel=True ):
Devin Lim142b5342017-07-20 15:22:39 -0700291 """
292 Description:
293 Installing onos.
294 Required:
295 * installMax - True for installing max number of nodes
296 False for installing current running nodes only.
297 Returns:
298 Returns main.TRUE if it successfully installed
299 """
300 result = main.TRUE
301 threads = []
302 i = 0
303 for ctrl in self.controllers if installMax else self.runningNodes:
304 options = "-f"
305 if installMax and i >= self.numCtrls:
306 options = "-nf"
Devin Lime9f0ccf2017-08-11 17:25:12 -0700307 if installParallel:
Jon Hall4173b242017-09-12 17:04:38 -0700308 t = main.Thread( target=ctrl.Bench.onosInstall,
309 name="install-" + ctrl.name,
310 kwargs={ "node" : ctrl.ipAddress,
311 "options" : options } )
Devin Lime9f0ccf2017-08-11 17:25:12 -0700312 threads.append( t )
313 t.start()
314 else:
315 result = result and \
Devin Lim142b5342017-07-20 15:22:39 -0700316 main.ONOSbench.onosInstall( node=ctrl.ipAddress, options=options )
Jon Halla6771b32018-01-26 16:07:28 -0800317 i += 1
Devin Lime9f0ccf2017-08-11 17:25:12 -0700318 if installParallel:
319 for t in threads:
320 t.join()
321 result = result and t.result
Jon Hallca319892017-06-15 15:25:22 -0700322 return result
323
324 def startCLIs( self ):
325 """
Devin Lim142b5342017-07-20 15:22:39 -0700326 Description:
327 starting Onos using onosCli driver
328 Required:
329 Returns:
330 Returns main.TRUE if it successfully started.
Jon Hallca319892017-06-15 15:25:22 -0700331 """
Devin Lim40a19092017-08-15 14:54:22 -0700332 result = main.TRUE
333 cliResults = self.command( "startOnosCli",
334 args=[ "ipAddress" ],
335 specificDriver=2,
336 getFrom=1,
337 funcFromCtrl=True )
Jon Hall4173b242017-09-12 17:04:38 -0700338 for i in range( len( cliResults ) ):
Devin Lim40a19092017-08-15 14:54:22 -0700339 result = result and cliResults[ i ]
340 self.controllers[ i ].active = True
341 return result
Jon Hallca319892017-06-15 15:25:22 -0700342
Devin Lim3ebd5e72017-11-14 10:38:00 -0800343 def nodesCheck( self ):
344 results = True
345 nodesOutput = self.command( "nodes", specificDriver=2 )
346 ips = sorted( self.getIps( activeOnly=True ) )
347 for i in nodesOutput:
348 try:
349 current = json.loads( i )
350 activeIps = []
351 currentResult = False
352 for node in current:
353 if node[ 'state' ] == 'READY':
354 activeIps.append( node[ 'ip' ] )
355 activeIps.sort()
356 if ips == activeIps:
357 currentResult = True
358 except ( ValueError, TypeError ):
359 main.log.error( "Error parsing nodes output" )
360 main.log.warn( repr( i ) )
361 currentResult = False
362 results = results and currentResult
363 return results
364
Devin Lim142b5342017-07-20 15:22:39 -0700365 def printResult( self, results, activeList, logLevel="debug" ):
366 """
367 Description:
368 Print the value of the list.
369 Required:
370 * results - list of the result
371 * activeList - list of the acitve nodes.
372 * logLevel - Type of log level you want it to be printed.
373 Returns:
374 """
375 f = getattr( main.log, logLevel )
Jon Hall4173b242017-09-12 17:04:38 -0700376 for i in range( len( results ) ):
377 f( activeList[ i ].name + "'s result : " + str( results[ i ] ) )
Devin Lim142b5342017-07-20 15:22:39 -0700378
379 def allTrueResultCheck( self, results, activeList ):
380 """
381 Description:
382 check if all the result has main.TRUE.
383 Required:
384 * results - list of the result
385 * activeList - list of the acitve nodes.
386 Returns:
387 Returns True if all == main.TRUE else
388 returns False
389 """
390 self.printResult( results, activeList )
391 return all( result == main.TRUE for result in results )
392
393 def notEmptyResultCheck( self, results, activeList ):
394 """
395 Description:
396 check if all the result has any contents
397 Required:
398 * results - list of the result
399 * activeList - list of the acitve nodes.
400 Returns:
401 Returns True if all the results has
402 something else returns False
403 """
404 self.printResult( results, activeList )
405 return all( result for result in results )
406
407 def identicalResultsCheck( self, results, activeList ):
408 """
409 Description:
410 check if all the results has same output.
411 Required:
412 * results - list of the result
413 * activeList - list of the acitve nodes.
414 Returns:
415 Returns True if all the results has
416 same result else returns False
417 """
418 self.printResult( results, activeList )
419 resultOne = results[ 0 ]
420 return all( resultOne == result for result in results )
421
Devin Lime9f0ccf2017-08-11 17:25:12 -0700422 def command( self, function, args=(), kwargs={}, returnBool=False,
Devin Lim40a19092017-08-15 14:54:22 -0700423 specificDriver=0, contentCheck=False, getFrom=2,
424 funcFromCtrl=False ):
Devin Lim142b5342017-07-20 15:22:39 -0700425 """
426 Description:
427 execute some function of the active nodes.
428 Required:
429 * function - name of the function
430 * args - argument of the function
431 * kwargs - kwargs of the funciton
432 * returnBool - True if wish to check all the result has main.TRUE
433 * specificDriver - specific driver to execute the function. Since
434 some of the function can be used in different drivers, it is important
435 to specify which driver it will be executed from.
436 0 - any type of driver
437 1 - from bench
438 2 - from cli
439 3 - from rest
440 * contentCheck - If this is True, it will check if the result has some
441 contents.
Devin Lime9f0ccf2017-08-11 17:25:12 -0700442 * getFrom - from which nodes
443 2 - active nodes
444 1 - current running nodes
445 0 - all nodes
Devin Lim40a19092017-08-15 14:54:22 -0700446 * funcFromCtrl - specific function of the args/kwargs
447 from each controller from the list of the controllers
Devin Lim142b5342017-07-20 15:22:39 -0700448 Returns:
449 Returns results if not returnBool and not contentCheck
450 Returns checkTruthValue of the result if returnBool
451 Returns resultContent of the result if contentCheck
452 """
Jon Hallca319892017-06-15 15:25:22 -0700453 threads = []
Devin Lim142b5342017-07-20 15:22:39 -0700454 drivers = [ None, "Bench", "CLI", "REST" ]
Devin Lime9f0ccf2017-08-11 17:25:12 -0700455 fromNode = [ self.controllers, self.runningNodes, self.active() ]
Jon Hallca319892017-06-15 15:25:22 -0700456 results = []
Devin Lime9f0ccf2017-08-11 17:25:12 -0700457 for ctrl in fromNode[ getFrom ]:
Devin Lim142b5342017-07-20 15:22:39 -0700458 try:
Devin Lim40a19092017-08-15 14:54:22 -0700459 funcArgs = []
460 funcKwargs = {}
Devin Lim142b5342017-07-20 15:22:39 -0700461 f = getattr( ( ctrl if not specificDriver else
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700462 getattr( ctrl, drivers[ specificDriver ] ) ), function )
Devin Lim40a19092017-08-15 14:54:22 -0700463 if funcFromCtrl:
464 if args:
465 for i in range( len( args ) ):
466 funcArgs.append( getattr( ctrl, args[ i ] ) )
467 if kwargs:
468 for k in kwargs:
Jon Hall4173b242017-09-12 17:04:38 -0700469 funcKwargs.update( { k: getattr( ctrl, kwargs[ k ] ) } )
Devin Lim142b5342017-07-20 15:22:39 -0700470 except AttributeError:
471 main.log.error( "Function " + function + " not found. Exiting the Test." )
Devin Lim44075962017-08-11 10:56:37 -0700472 main.cleanAndExit()
Jon Hallca319892017-06-15 15:25:22 -0700473 t = main.Thread( target=f,
474 name=function + "-" + ctrl.name,
Devin Lim40a19092017-08-15 14:54:22 -0700475 args=funcArgs if funcFromCtrl else args,
476 kwargs=funcKwargs if funcFromCtrl else kwargs )
Jon Hallca319892017-06-15 15:25:22 -0700477 threads.append( t )
478 t.start()
479
480 for t in threads:
481 t.join()
482 results.append( t.result )
Devin Lim142b5342017-07-20 15:22:39 -0700483 if returnBool:
Devin Lime9f0ccf2017-08-11 17:25:12 -0700484 return self.allTrueResultCheck( results, fromNode[ getFrom ] )
Devin Lim142b5342017-07-20 15:22:39 -0700485 elif contentCheck:
Devin Lime9f0ccf2017-08-11 17:25:12 -0700486 return self.notEmptyResultCheck( results, fromNode[ getFrom ] )
Jon Hall4173b242017-09-12 17:04:38 -0700487 return results
488
489 def checkPartitionSize( self, segmentSize='64', units='M', multiplier='3' ):
490 # max segment size in bytes: 1024 * 1024 * 64
491 # multiplier is somewhat arbitrary, but the idea is the logs would have
492 # been compacted before this many segments are written
493
494 maxSize = float( segmentSize ) * float( multiplier )
495 ret = True
496 for n in self.runningNodes:
Jon Hall5d5876e2017-11-30 09:33:16 -0800497 # Partition logs
498 ret = ret and n.server.folderSize( "/opt/onos/apache-karaf-*/data/db/partitions/*/*.log",
Jon Hall4173b242017-09-12 17:04:38 -0700499 size=maxSize, unit=units, ignoreRoot=False )
500 return ret