Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 1 | """ |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 2 | Copyright 2017 Open Networking Foundation ( ONF ) |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 3 | |
| 4 | Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>, |
| 5 | the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>, |
| 6 | or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg> |
| 7 | |
| 8 | TestON is free software: you can redistribute it and/or modify |
| 9 | it under the terms of the GNU General Public License as published by |
| 10 | the Free Software Foundation, either version 2 of the License, or |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 11 | ( at your option ) any later version. |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 12 | |
| 13 | TestON is distributed in the hope that it will be useful, |
| 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | GNU General Public License for more details. |
| 17 | |
| 18 | You should have received a copy of the GNU General Public License |
| 19 | along with TestON. If not, see <http://www.gnu.org/licenses/>. |
| 20 | """ |
Devin Lim | 3ebd5e7 | 2017-11-14 10:38:00 -0800 | [diff] [blame] | 21 | import json |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 22 | class Cluster(): |
| 23 | |
| 24 | def __str__( self ): |
| 25 | return self.name |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 26 | |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 27 | def __repr__( self ): |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 28 | controllers = [] |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 29 | runningNodes = self.getRunningNodes() |
| 30 | atomixNodes = [] |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 31 | for ctrl in self.controllers: |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 32 | controllers.append( "{%s:%s, %s - %s}" % ( ctrl.name, |
| 33 | ctrl.ipAddress, |
| 34 | "Configured" if ctrl in runningNodes else "Not Configured", |
| 35 | "Active" if ctrl.active else "Inactive" ) ) |
| 36 | for node in self.atomixNodes: |
| 37 | atomixNodes.append( "{%s:%s}" % ( node.name, node.ipAddress ) ) |
| 38 | return "%s[%s; Atomix Nodes:%s]" % ( self.name, ", ".join( controllers ), ", ".join( atomixNodes ) ) |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 39 | |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 40 | def __init__( self, ctrlList=[], name="Cluster" ): |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 41 | """ |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 42 | controllers : All the nodes |
| 43 | runningNodes : Node that are specifically running from the test. |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 44 | ie ) When the test is testing different number of nodes on each |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 45 | run. |
| 46 | numCtrls : number of runningNodes |
| 47 | maxCtrls : number of controllers |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 48 | """ |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 49 | self.controllers = ctrlList |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 50 | self.runningNodes = ctrlList |
| 51 | self.numCtrls = len( self.runningNodes ) |
| 52 | self.maxCtrls = len( self.controllers ) |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 53 | self.name = str( name ) |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 54 | self.atomixNodes = ctrlList |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 55 | self.iterator = iter( self.active() ) |
| 56 | |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 57 | def fromNode( self, ctrlList ): |
| 58 | """ |
| 59 | Helper function to get a specific list of controllers |
| 60 | Required Arguments: |
| 61 | * ctrlList - The list of controllers to return. This can be either an index or a keyword |
| 62 | Index | Keyword | List returned |
| 63 | 0 | "all" | self.controllers |
| 64 | 1 | "running" | self.runningNodes |
| 65 | 2 | "active | self.active() |
| 66 | |
| 67 | Throws a ValueError exception if ctrlList value is not recognized |
| 68 | """ |
| 69 | # TODO: Add Atomix Nodes? |
| 70 | if isinstance( ctrlList, str ): |
| 71 | ctrlList = ctrlList.lower() |
| 72 | if ctrlList == 0 or ctrlList == "all": |
| 73 | return self.controllers |
| 74 | elif ctrlList == 1 or ctrlList == "running": |
| 75 | return self.runningNodes |
| 76 | elif ctrlList == 2 or ctrlList == "active": |
| 77 | return self.active() |
| 78 | else: |
| 79 | raise ValueError( "Unknown argument: {}".format( ctrlList ) ) |
| 80 | |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 81 | def getIps( self, activeOnly=False, allNode=False ): |
| 82 | """ |
| 83 | Description: |
| 84 | get the list of the ip. Default to get ips of running nodes. |
| 85 | Required: |
| 86 | * activeOnly - True for getting ips of active nodes |
| 87 | * allNode - True for getting ips of all nodes |
| 88 | Returns: |
| 89 | Retruns ips of active nodes if activeOnly is True. |
| 90 | Returns ips of all nodes if allNode is True. |
| 91 | """ |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 92 | ips = [] |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 93 | if allNode: |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 94 | nodeList = self.controllers |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 95 | else: |
| 96 | if activeOnly: |
| 97 | nodeList = self.active() |
| 98 | else: |
| 99 | nodeList = self.runningNodes |
| 100 | |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 101 | for ctrl in nodeList: |
| 102 | ips.append( ctrl.ipAddress ) |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 103 | |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 104 | return ips |
| 105 | |
Jon Hall | a1e8e51 | 2018-05-11 13:30:57 -0700 | [diff] [blame] | 106 | def clearActive( self ): |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 107 | """ |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 108 | Description: |
Jon Hall | a1e8e51 | 2018-05-11 13:30:57 -0700 | [diff] [blame] | 109 | Sets the activeness of each cluster node to be False |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 110 | """ |
Jon Hall | ab61137 | 2018-02-21 15:26:05 -0800 | [diff] [blame] | 111 | for ctrl in self.controllers: |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 112 | ctrl.active = False |
| 113 | |
| 114 | def getRunningPos( self ): |
| 115 | """ |
| 116 | Description: |
| 117 | get the position of the active running nodes. |
| 118 | Required: |
| 119 | Returns: |
| 120 | Retruns the list of the position of the active |
| 121 | running nodes. |
| 122 | """ |
| 123 | return [ ctrl.pos for ctrl in self.runningNodes |
| 124 | if ctrl.active ] |
| 125 | |
| 126 | def setRunningNode( self, numCtrls ): |
| 127 | """ |
| 128 | Description: |
| 129 | Set running nodes of n number of nodes. |
| 130 | It will create new list of runningNodes. |
| 131 | If numCtrls is a list, it will add the nodes of the |
| 132 | list. |
| 133 | Required: |
| 134 | * numCtrls - number of nodes to be set. |
| 135 | Returns: |
| 136 | """ |
| 137 | self.runningNodes = [] |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 138 | for i in numCtrls if isinstance( numCtrls, list ) else range( numCtrls ): |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 139 | self.runningNodes.append( self.controllers[ i ] ) |
| 140 | self.numCtrls = len( numCtrls ) if isinstance( numCtrls, list ) else numCtrls |
| 141 | |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 142 | def setAtomixNodes( self, nodes ): |
| 143 | """ |
| 144 | Description: |
| 145 | Sets the list of Atomix nodes for the cluster |
| 146 | If nodes is a list, it will add the nodes of the list. |
| 147 | If nodes is an int, the function will set the Atomix nodes |
| 148 | to be the first n in the list of controllers. |
| 149 | Required: |
| 150 | * nodes - number of nodes to be set, or a list of nodes to set |
| 151 | Returns: |
| 152 | """ |
| 153 | self.atomixNodes = [] |
| 154 | for i in nodes if isinstance( nodes, list ) else range( nodes ): |
| 155 | self.atomixNodes.append( self.controllers[ i ] ) |
| 156 | |
| 157 | def getControllers( self, node=None ): |
| 158 | """ |
| 159 | Description: |
| 160 | Get the list of all controllers in a cluster or a controller at an index in the list |
| 161 | Optional Arguments: |
| 162 | * node - position of the node to get from the list of controllers. |
| 163 | Returns: |
| 164 | Return a list of controllers in the cluster if node is None |
| 165 | if not, it will return the controller at the given index. |
| 166 | """ |
| 167 | result = self.controllers |
| 168 | return result if node is None else result[ node % len( result ) ] |
| 169 | |
| 170 | def getRunningNodes( self, node=None ): |
| 171 | """ |
| 172 | Description: |
| 173 | Get the list of all controllers in a cluster that should be running or |
| 174 | a controller at an index in the list |
| 175 | Optional Arguments: |
| 176 | * node - position of the node to get from the list of controllers. |
| 177 | Returns: |
| 178 | Return a list of controllers in the cluster if node is None |
| 179 | if not, it will return the controller at the given index. |
| 180 | """ |
| 181 | result = self.runningNodes |
| 182 | return result if node is None else result[ node % len( result ) ] |
| 183 | |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 184 | def active( self, node=None ): |
| 185 | """ |
| 186 | Description: |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 187 | Get the list of all active controllers in a cluster or |
| 188 | a controller at an index in the list |
| 189 | Optional Arguments: |
| 190 | * node - position of the node to get from the list of controllers. |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 191 | Returns: |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 192 | Return a list of controllers in the cluster if node is None |
| 193 | if not, it will return the controller at the given index. |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 194 | """ |
| 195 | result = [ ctrl for ctrl in self.runningNodes |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 196 | if ctrl.active ] |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 197 | return result if node is None else result[ node % len( result ) ] |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 198 | |
| 199 | def next( self ): |
| 200 | """ |
Jon Hall | a1e8e51 | 2018-05-11 13:30:57 -0700 | [diff] [blame] | 201 | An iterator for the cluster's active controllers that |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 202 | resets when there are no more elements. |
| 203 | |
| 204 | Returns the next controller in the cluster |
| 205 | """ |
| 206 | try: |
Jon Hall | a1e8e51 | 2018-05-11 13:30:57 -0700 | [diff] [blame] | 207 | node = self.iterator.next() |
| 208 | assert node.active |
| 209 | return node |
| 210 | except ( StopIteration, AssertionError ): |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 211 | self.reset() |
| 212 | try: |
| 213 | return self.iterator.next() |
| 214 | except StopIteration: |
| 215 | raise RuntimeError( "There are no active nodes in the cluster" ) |
| 216 | |
| 217 | def reset( self ): |
| 218 | """ |
| 219 | Resets the cluster iterator. |
| 220 | |
| 221 | This should be used whenever a node's active state is changed |
| 222 | and is also used internally when the iterator has been exhausted. |
| 223 | """ |
| 224 | self.iterator = iter( self.active() ) |
| 225 | |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 226 | def createCell( self, cellName, cellApps, mininetIp, useSSH, onosIps, |
| 227 | atomixIps, installMax=False ): |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 228 | """ |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 229 | Description: |
| 230 | create a new cell |
| 231 | Required: |
| 232 | * cellName - The name of the cell. |
You Wang | 09b596b | 2018-01-10 10:42:38 -0800 | [diff] [blame] | 233 | * cellApps - The ONOS apps string. |
You Wang | a0f6ff6 | 2018-01-11 15:46:30 -0800 | [diff] [blame] | 234 | * mininetIp - Mininet IP address. |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 235 | * useSSH - True for using ssh when creating a cell |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 236 | * onosIps - ip( s ) of the ONOS node( s ). |
| 237 | * atomixIps - ip( s ) of the Atomix node( s ). |
| 238 | |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 239 | Returns: |
| 240 | """ |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 241 | self.command( "createCellFile", |
| 242 | args=[ main.ONOSbench.ip_address, |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 243 | cellName, |
You Wang | a0f6ff6 | 2018-01-11 15:46:30 -0800 | [diff] [blame] | 244 | mininetIp, |
You Wang | 0d9f2c0 | 2018-08-10 14:56:32 -0700 | [diff] [blame] | 245 | cellApps, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 246 | onosIps, |
| 247 | atomixIps, |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 248 | main.ONOScell.karafUser, |
| 249 | useSSH ], |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 250 | specificDriver=1, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 251 | getFrom="all" if installMax else "running" ) |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 252 | |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 253 | def uninstallAtomix( self, uninstallMax ): |
| 254 | """ |
| 255 | Description: |
| 256 | uninstalling atomix |
| 257 | Required: |
| 258 | * uninstallMax - True for uninstalling max number of nodes |
| 259 | False for uninstalling the current running nodes. |
| 260 | Returns: |
| 261 | Returns main.TRUE if it successfully uninstalled. |
| 262 | """ |
| 263 | result = main.TRUE |
| 264 | uninstallResult = self.command( "atomixUninstall", |
| 265 | kwargs={ "nodeIp": "ipAddress" }, |
| 266 | specificDriver=1, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 267 | getFrom="all" if uninstallMax else "running", |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 268 | funcFromCtrl=True ) |
| 269 | for uninstallR in uninstallResult: |
| 270 | result = result and uninstallR |
| 271 | return result |
| 272 | |
| 273 | def uninstallOnos( self, uninstallMax ): |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 274 | """ |
| 275 | Description: |
| 276 | uninstalling onos |
| 277 | Required: |
| 278 | * uninstallMax - True for uninstalling max number of nodes |
| 279 | False for uninstalling the current running nodes. |
| 280 | Returns: |
| 281 | Returns main.TRUE if it successfully uninstalled. |
| 282 | """ |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 283 | result = main.TRUE |
| 284 | uninstallResult = self.command( "onosUninstall", |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 285 | kwargs={ "nodeIp": "ipAddress" }, |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 286 | specificDriver=1, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 287 | getFrom="all" if uninstallMax else "running", |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 288 | funcFromCtrl=True ) |
| 289 | for uninstallR in uninstallResult: |
| 290 | result = result and uninstallR |
| 291 | return result |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 292 | |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 293 | def applyCell( self, cellName, installMax=False ): |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 294 | """ |
| 295 | Description: |
| 296 | apply the cell with cellName. It will also verify the |
| 297 | cell. |
| 298 | Required: |
| 299 | * cellName - The name of the cell. |
| 300 | Returns: |
| 301 | Returns main.TRUE if it successfully set and verify cell. |
| 302 | """ |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 303 | setCellResult = self.command( "setCell", |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 304 | args=[ cellName ], |
| 305 | specificDriver=1, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 306 | getFrom="all" ) |
| 307 | benchCellResult = main.ONOSbench.setCell( cellName ) |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 308 | verifyResult = self.command( "verifyCell", |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 309 | specificDriver=1, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 310 | getFrom="all" ) |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 311 | result = main.TRUE |
| 312 | for i in range( len( setCellResult ) ): |
| 313 | result = result and setCellResult[ i ] and verifyResult[ i ] |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 314 | result = result and benchCellResult |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 315 | return result |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 316 | |
| 317 | def checkService( self ): |
| 318 | """ |
| 319 | Description: |
| 320 | Checking if the onos service is up. If not, it will |
| 321 | start the onos service manually. |
| 322 | Required: |
| 323 | Returns: |
| 324 | Returns main.TRUE if it successfully checked |
| 325 | """ |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 326 | getFrom = "running" |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 327 | onosIsUp = main.TRUE |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 328 | onosUp = self.command( "isup", |
| 329 | args=[ "ipAddress" ], |
| 330 | specificDriver=1, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 331 | getFrom=getFrom, |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 332 | funcFromCtrl=True ) |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 333 | ctrlList = self.fromNode( getFrom ) |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 334 | for i in range( len( onosUp ) ): |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 335 | ctrl = ctrlList[ i ] |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 336 | onosIsUp = onosIsUp and onosUp[ i ] |
| 337 | if onosUp[ i ] == main.TRUE: |
Jon Hall | 9655dcb | 2018-02-02 11:19:06 -0800 | [diff] [blame] | 338 | main.log.info( ctrl.name + " is up and ready" ) |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 339 | else: |
Jon Hall | 9655dcb | 2018-02-02 11:19:06 -0800 | [diff] [blame] | 340 | main.log.warn( ctrl.name + " may not be up." ) |
| 341 | return onosIsUp |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 342 | |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 343 | def killAtomix( self, killMax, stopAtomix ): |
| 344 | """ |
| 345 | Description: |
| 346 | killing atomix. It will either kill the current runningnodes or |
| 347 | max number of the nodes. |
| 348 | Required: |
| 349 | * killRemoveMax - The boolean that will decide either to kill |
| 350 | only running nodes ( False ) or max number of nodes ( True ). |
| 351 | * stopAtomix - If wish to atomix onos before killing it. True for |
| 352 | enable stop, False for disable stop. |
| 353 | Returns: |
| 354 | Returns main.TRUE if successfully killing it. |
| 355 | """ |
| 356 | result = main.TRUE |
| 357 | killResult = self.command( "atomixKill", |
| 358 | args=[ "ipAddress" ], |
| 359 | specificDriver=1, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 360 | getFrom="all" if killMax else "running", |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 361 | funcFromCtrl=True ) |
| 362 | for i in range( len( killResult ) ): |
| 363 | result = result and killResult[ i ] |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 364 | return result |
| 365 | |
| 366 | def killOnos( self, killMax, stopOnos ): |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 367 | """ |
| 368 | Description: |
| 369 | killing the onos. It will either kill the current runningnodes or |
| 370 | max number of the nodes. |
| 371 | Required: |
| 372 | * killRemoveMax - The boolean that will decide either to kill |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 373 | only running nodes ( False ) or max number of nodes ( True ). |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 374 | * stopOnos - If wish to stop onos before killing it. True for |
| 375 | enable stop , False for disable stop. |
| 376 | Returns: |
| 377 | Returns main.TRUE if successfully killing it. |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 378 | """ |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 379 | getFrom = "all" if killMax else "running" |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 380 | result = main.TRUE |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 381 | killResult = self.command( "onosKill", |
| 382 | args=[ "ipAddress" ], |
| 383 | specificDriver=1, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 384 | getFrom=getFrom, |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 385 | funcFromCtrl=True ) |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 386 | ctrlList = self.fromNode( getFrom ) |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 387 | for i in range( len( killResult ) ): |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 388 | result = result and killResult[ i ] |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 389 | ctrlList[ i ].active = False |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 390 | return result |
| 391 | |
| 392 | def ssh( self ): |
| 393 | """ |
| 394 | Description: |
| 395 | set up ssh to the onos |
| 396 | Required: |
| 397 | Returns: |
| 398 | Returns main.TRUE if it successfully setup the ssh to |
| 399 | the onos. |
| 400 | """ |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 401 | result = main.TRUE |
| 402 | sshResult = self.command( "onosSecureSSH", |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 403 | kwargs={ "node": "ipAddress" }, |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 404 | specificDriver=1, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 405 | getFrom="running", |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 406 | funcFromCtrl=True ) |
| 407 | for sshR in sshResult: |
| 408 | result = result and sshR |
| 409 | return result |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 410 | |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 411 | def installAtomix( self, installParallel=True ): |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 412 | """ |
| 413 | Description: |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 414 | Installing Atomix. |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 415 | Required: |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 416 | Returns: |
| 417 | Returns main.TRUE if it successfully installed |
| 418 | """ |
| 419 | result = main.TRUE |
| 420 | threads = [] |
| 421 | i = 0 |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 422 | for ctrl in self.atomixNodes: |
| 423 | options = "-f" |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 424 | if installParallel: |
| 425 | t = main.Thread( target=ctrl.Bench.atomixInstall, |
| 426 | name="atomix-install-" + ctrl.name, |
| 427 | kwargs={ "node" : ctrl.ipAddress, |
| 428 | "options" : options } ) |
| 429 | threads.append( t ) |
| 430 | t.start() |
| 431 | else: |
| 432 | result = result and \ |
| 433 | main.ONOSbench.atomixInstall( node=ctrl.ipAddress, options=options ) |
| 434 | i += 1 |
| 435 | if installParallel: |
| 436 | for t in threads: |
| 437 | t.join() |
| 438 | result = result and t.result |
| 439 | return result |
| 440 | |
| 441 | def installOnos( self, installMax=True, installParallel=True ): |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 442 | """ |
| 443 | Description: |
| 444 | Installing onos. |
| 445 | Required: |
| 446 | * installMax - True for installing max number of nodes |
| 447 | False for installing current running nodes only. |
| 448 | Returns: |
| 449 | Returns main.TRUE if it successfully installed |
| 450 | """ |
| 451 | result = main.TRUE |
| 452 | threads = [] |
| 453 | i = 0 |
| 454 | for ctrl in self.controllers if installMax else self.runningNodes: |
| 455 | options = "-f" |
| 456 | if installMax and i >= self.numCtrls: |
| 457 | options = "-nf" |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 458 | if installParallel: |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 459 | t = main.Thread( target=ctrl.Bench.onosInstall, |
You Wang | f9d95be | 2018-08-01 14:35:37 -0700 | [diff] [blame] | 460 | name="onos-install-" + ctrl.name, |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 461 | kwargs={ "node" : ctrl.ipAddress, |
| 462 | "options" : options } ) |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 463 | threads.append( t ) |
| 464 | t.start() |
| 465 | else: |
| 466 | result = result and \ |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 467 | main.ONOSbench.onosInstall( node=ctrl.ipAddress, options=options ) |
Jon Hall | a6771b3 | 2018-01-26 16:07:28 -0800 | [diff] [blame] | 468 | i += 1 |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 469 | if installParallel: |
| 470 | for t in threads: |
| 471 | t.join() |
| 472 | result = result and t.result |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 473 | return result |
| 474 | |
| 475 | def startCLIs( self ): |
| 476 | """ |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 477 | Description: |
| 478 | starting Onos using onosCli driver |
| 479 | Required: |
| 480 | Returns: |
| 481 | Returns main.TRUE if it successfully started. |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 482 | """ |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 483 | getFrom = "running" |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 484 | result = main.TRUE |
| 485 | cliResults = self.command( "startOnosCli", |
| 486 | args=[ "ipAddress" ], |
| 487 | specificDriver=2, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 488 | getFrom=getFrom, |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 489 | funcFromCtrl=True ) |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 490 | ctrlList = self.fromNode( getFrom ) |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 491 | for i in range( len( cliResults ) ): |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 492 | result = result and cliResults[ i ] |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 493 | ctrlList[ i ].active = True |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 494 | return result |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 495 | |
Devin Lim | 3ebd5e7 | 2017-11-14 10:38:00 -0800 | [diff] [blame] | 496 | def nodesCheck( self ): |
You Wang | 0d9f2c0 | 2018-08-10 14:56:32 -0700 | [diff] [blame] | 497 | """ |
| 498 | Description: |
| 499 | Checking if all the onos nodes are in READY state |
| 500 | Required: |
| 501 | Returns: |
| 502 | Returns True if it successfully checked |
| 503 | """ |
Devin Lim | 3ebd5e7 | 2017-11-14 10:38:00 -0800 | [diff] [blame] | 504 | results = True |
| 505 | nodesOutput = self.command( "nodes", specificDriver=2 ) |
| 506 | ips = sorted( self.getIps( activeOnly=True ) ) |
| 507 | for i in nodesOutput: |
| 508 | try: |
| 509 | current = json.loads( i ) |
| 510 | activeIps = [] |
| 511 | currentResult = False |
| 512 | for node in current: |
| 513 | if node[ 'state' ] == 'READY': |
| 514 | activeIps.append( node[ 'ip' ] ) |
| 515 | activeIps.sort() |
| 516 | if ips == activeIps: |
| 517 | currentResult = True |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 518 | else: |
| 519 | main.log.error( "{} != {}".format( ips, activeIps ) ) |
Devin Lim | 3ebd5e7 | 2017-11-14 10:38:00 -0800 | [diff] [blame] | 520 | except ( ValueError, TypeError ): |
| 521 | main.log.error( "Error parsing nodes output" ) |
| 522 | main.log.warn( repr( i ) ) |
| 523 | currentResult = False |
| 524 | results = results and currentResult |
| 525 | return results |
| 526 | |
You Wang | 0d9f2c0 | 2018-08-10 14:56:32 -0700 | [diff] [blame] | 527 | def appsCheck( self, apps ): |
| 528 | """ |
| 529 | Description: |
| 530 | Checking if all the applications are activated |
| 531 | Required: |
| 532 | apps: list of applications that are expected to be activated |
| 533 | Returns: |
| 534 | Returns True if it successfully checked |
| 535 | """ |
| 536 | results = True |
| 537 | for app in apps: |
| 538 | states = self.command( "appStatus", |
| 539 | args=[ app ], |
| 540 | specificDriver=2 ) |
| 541 | for i in range( len( states ) ): |
| 542 | ctrl = self.controllers[ i ] |
| 543 | if states[ i ] == "ACTIVE": |
| 544 | results = results and True |
| 545 | main.log.info( "{}: {} is activated".format( ctrl.name, app ) ) |
| 546 | else: |
| 547 | results = False |
| 548 | main.log.warn( "{}: {} is in {} state".format( ctrl.name, app, states[ i ] ) ) |
| 549 | return results |
| 550 | |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 551 | def printResult( self, results, activeList, logLevel="debug" ): |
| 552 | """ |
| 553 | Description: |
| 554 | Print the value of the list. |
| 555 | Required: |
| 556 | * results - list of the result |
| 557 | * activeList - list of the acitve nodes. |
| 558 | * logLevel - Type of log level you want it to be printed. |
| 559 | Returns: |
| 560 | """ |
| 561 | f = getattr( main.log, logLevel ) |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 562 | for i in range( len( results ) ): |
| 563 | f( activeList[ i ].name + "'s result : " + str( results[ i ] ) ) |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 564 | |
| 565 | def allTrueResultCheck( self, results, activeList ): |
| 566 | """ |
| 567 | Description: |
| 568 | check if all the result has main.TRUE. |
| 569 | Required: |
| 570 | * results - list of the result |
| 571 | * activeList - list of the acitve nodes. |
| 572 | Returns: |
| 573 | Returns True if all == main.TRUE else |
| 574 | returns False |
| 575 | """ |
| 576 | self.printResult( results, activeList ) |
| 577 | return all( result == main.TRUE for result in results ) |
| 578 | |
| 579 | def notEmptyResultCheck( self, results, activeList ): |
| 580 | """ |
| 581 | Description: |
| 582 | check if all the result has any contents |
| 583 | Required: |
| 584 | * results - list of the result |
| 585 | * activeList - list of the acitve nodes. |
| 586 | Returns: |
| 587 | Returns True if all the results has |
| 588 | something else returns False |
| 589 | """ |
| 590 | self.printResult( results, activeList ) |
| 591 | return all( result for result in results ) |
| 592 | |
| 593 | def identicalResultsCheck( self, results, activeList ): |
| 594 | """ |
| 595 | Description: |
| 596 | check if all the results has same output. |
| 597 | Required: |
| 598 | * results - list of the result |
| 599 | * activeList - list of the acitve nodes. |
| 600 | Returns: |
| 601 | Returns True if all the results has |
| 602 | same result else returns False |
| 603 | """ |
| 604 | self.printResult( results, activeList ) |
| 605 | resultOne = results[ 0 ] |
| 606 | return all( resultOne == result for result in results ) |
| 607 | |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 608 | def command( self, function, args=(), kwargs={}, returnBool=False, |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 609 | specificDriver=0, contentCheck=False, getFrom="active", |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 610 | funcFromCtrl=False ): |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 611 | """ |
| 612 | Description: |
| 613 | execute some function of the active nodes. |
| 614 | Required: |
| 615 | * function - name of the function |
| 616 | * args - argument of the function |
| 617 | * kwargs - kwargs of the funciton |
| 618 | * returnBool - True if wish to check all the result has main.TRUE |
| 619 | * specificDriver - specific driver to execute the function. Since |
| 620 | some of the function can be used in different drivers, it is important |
| 621 | to specify which driver it will be executed from. |
| 622 | 0 - any type of driver |
| 623 | 1 - from bench |
| 624 | 2 - from cli |
| 625 | 3 - from rest |
| 626 | * contentCheck - If this is True, it will check if the result has some |
| 627 | contents. |
Devin Lim | e9f0ccf | 2017-08-11 17:25:12 -0700 | [diff] [blame] | 628 | * getFrom - from which nodes |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 629 | 2 or "active" - active nodes |
| 630 | 1 or "running" - current running nodes |
| 631 | 0 or "all" - all nodes |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 632 | * funcFromCtrl - specific function of the args/kwargs |
| 633 | from each controller from the list of the controllers |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 634 | Returns: |
| 635 | Returns results if not returnBool and not contentCheck |
| 636 | Returns checkTruthValue of the result if returnBool |
| 637 | Returns resultContent of the result if contentCheck |
| 638 | """ |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 639 | threads = [] |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 640 | drivers = [ None, "Bench", "CLI", "REST" ] |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 641 | results = [] |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 642 | for ctrl in self.fromNode( getFrom ): |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 643 | try: |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 644 | funcArgs = [] |
| 645 | funcKwargs = {} |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 646 | f = getattr( ( ctrl if not specificDriver else |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 647 | getattr( ctrl, drivers[ specificDriver ] ) ), function ) |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 648 | if funcFromCtrl: |
| 649 | if args: |
| 650 | for i in range( len( args ) ): |
| 651 | funcArgs.append( getattr( ctrl, args[ i ] ) ) |
| 652 | if kwargs: |
| 653 | for k in kwargs: |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 654 | funcKwargs.update( { k: getattr( ctrl, kwargs[ k ] ) } ) |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 655 | except AttributeError: |
| 656 | main.log.error( "Function " + function + " not found. Exiting the Test." ) |
Devin Lim | 4407596 | 2017-08-11 10:56:37 -0700 | [diff] [blame] | 657 | main.cleanAndExit() |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 658 | t = main.Thread( target=f, |
| 659 | name=function + "-" + ctrl.name, |
Devin Lim | 40a1909 | 2017-08-15 14:54:22 -0700 | [diff] [blame] | 660 | args=funcArgs if funcFromCtrl else args, |
| 661 | kwargs=funcKwargs if funcFromCtrl else kwargs ) |
Jon Hall | ca31989 | 2017-06-15 15:25:22 -0700 | [diff] [blame] | 662 | threads.append( t ) |
| 663 | t.start() |
| 664 | |
| 665 | for t in threads: |
| 666 | t.join() |
| 667 | results.append( t.result ) |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 668 | if returnBool: |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 669 | return self.allTrueResultCheck( results, self.fromNode( getFrom ) ) |
Devin Lim | 142b534 | 2017-07-20 15:22:39 -0700 | [diff] [blame] | 670 | elif contentCheck: |
Jon Hall | 3e6edb3 | 2018-08-21 16:20:30 -0700 | [diff] [blame] | 671 | return self.notEmptyResultCheck( results, self.fromNode( getFrom ) ) |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 672 | return results |
| 673 | |
| 674 | def checkPartitionSize( self, segmentSize='64', units='M', multiplier='3' ): |
| 675 | # max segment size in bytes: 1024 * 1024 * 64 |
| 676 | # multiplier is somewhat arbitrary, but the idea is the logs would have |
| 677 | # been compacted before this many segments are written |
| 678 | |
| 679 | maxSize = float( segmentSize ) * float( multiplier ) |
| 680 | ret = True |
| 681 | for n in self.runningNodes: |
Jon Hall | 5d5876e | 2017-11-30 09:33:16 -0800 | [diff] [blame] | 682 | # Partition logs |
Jon Hall | d81e016 | 2018-08-13 12:06:13 -0700 | [diff] [blame] | 683 | ret = ret and n.server.folderSize( "/opt/atomix/data/raft/partitions/*/*.log", |
Jon Hall | 4173b24 | 2017-09-12 17:04:38 -0700 | [diff] [blame] | 684 | size=maxSize, unit=units, ignoreRoot=False ) |
| 685 | return ret |