You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 1 | """ |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 2 | Copyright 2016 Open Networking Foundation ( ONF ) |
Jeremy Ronquillo | b27ce4c | 2017-07-17 12:41:28 -0700 | [diff] [blame] | 3 | |
| 4 | Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>, |
| 5 | the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>, |
| 6 | or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg> |
| 7 | |
| 8 | TestON is free software: you can redistribute it and/or modify |
| 9 | it under the terms of the GNU General Public License as published by |
| 10 | the Free Software Foundation, either version 2 of the License, or |
Jeremy Ronquillo | 23fb216 | 2017-09-15 14:59:57 -0700 | [diff] [blame] | 11 | ( at your option ) any later version. |
Jeremy Ronquillo | b27ce4c | 2017-07-17 12:41:28 -0700 | [diff] [blame] | 12 | |
| 13 | TestON is distributed in the hope that it will be useful, |
| 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | GNU General Public License for more details. |
| 17 | |
| 18 | You should have received a copy of the GNU General Public License |
| 19 | along with TestON. If not, see <http://www.gnu.org/licenses/>. |
| 20 | """ |
Jeremy Ronquillo | b27ce4c | 2017-07-17 12:41:28 -0700 | [diff] [blame] | 21 | """ |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 22 | This file contains classes for CHOTestMonkey that are related to check event |
| 23 | Author: you@onlab.us |
| 24 | """ |
| 25 | from tests.CHOTestMonkey.dependencies.events.Event import EventType, EventStates, Event |
| 26 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 27 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 28 | class CheckEvent( Event ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 29 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 30 | def __init__( self ): |
| 31 | Event.__init__( self ) |
| 32 | |
| 33 | def startCheckEvent( self ): |
| 34 | return EventStates().PASS |
| 35 | |
| 36 | def startEvent( self, args ): |
| 37 | with self.eventLock: |
You Wang | 5216320 | 2016-07-14 16:37:15 -0700 | [diff] [blame] | 38 | main.log.info( "Event recorded: {} {}".format( self.typeIndex, self.typeString ) ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 39 | result = self.startCheckEvent() |
| 40 | return result |
| 41 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 42 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 43 | class IntentCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 44 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 45 | def __init__( self ): |
| 46 | CheckEvent.__init__( self ) |
| 47 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 48 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 49 | |
| 50 | def startCheckEvent( self, args=None ): |
| 51 | checkResult = EventStates().PASS |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 52 | intentDict = {} |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 53 | for intent in main.intents: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 54 | intentDict[ intent.id ] = intent.expectedState |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 55 | for controller in main.controllers: |
| 56 | if controller.isUp(): |
| 57 | with controller.CLILock: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 58 | intentState = controller.CLI.compareIntent( intentDict ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 59 | if not intentState: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 60 | main.log.warn( "Intent Check - not all intent ids and states match that on ONOS%s" % ( controller.index ) ) |
You Wang | 817990a | 2019-01-22 15:07:25 -0800 | [diff] [blame] | 61 | # FIXME: ONOS leaves intents as WITHDRAWN state occasionally and we don't consider that as a FAIL for now |
| 62 | # checkResult = EventStates().FAIL |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 63 | return checkResult |
| 64 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 65 | |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 66 | class FlowCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 67 | |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 68 | def __init__( self ): |
| 69 | CheckEvent.__init__( self ) |
| 70 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 71 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 72 | |
| 73 | def startCheckEvent( self, args=None ): |
| 74 | import json |
| 75 | checkResult = EventStates().PASS |
You Wang | 3c27625 | 2016-09-21 15:21:36 -0700 | [diff] [blame] | 76 | if main.enableIPv6: |
You Wang | 62e1cf6 | 2016-09-22 17:13:03 -0700 | [diff] [blame] | 77 | coreFlowNum = int( main.params[ 'EVENT' ][ 'FlowCheck' ][ 'coreFlowNum6' ] ) |
You Wang | 3c27625 | 2016-09-21 15:21:36 -0700 | [diff] [blame] | 78 | else: |
You Wang | 62e1cf6 | 2016-09-22 17:13:03 -0700 | [diff] [blame] | 79 | coreFlowNum = int( main.params[ 'EVENT' ][ 'FlowCheck' ][ 'coreFlowNum' ] ) |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 80 | for controller in main.controllers: |
| 81 | if controller.isUp(): |
| 82 | with controller.CLILock: |
You Wang | 3c27625 | 2016-09-21 15:21:36 -0700 | [diff] [blame] | 83 | # Check core flow number |
| 84 | for device in main.devices: |
| 85 | if device.isRemoved(): |
| 86 | continue |
| 87 | coreFlowNumOnos = controller.CLI.flowAddedCount( device.dpid, core=True ) |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 88 | if coreFlowNumOnos is None: |
You Wang | 3c27625 | 2016-09-21 15:21:36 -0700 | [diff] [blame] | 89 | main.log.warn( "Flow Check - error when trying to get flow number of %s on ONOS%s" % ( device.dpid, controller.index ) ) |
| 90 | checkResult = EventStates().FAIL |
| 91 | else: |
| 92 | coreFlowNumOnos = int( coreFlowNumOnos ) |
| 93 | if coreFlowNumOnos != coreFlowNum: |
| 94 | main.log.warn( "Flow Check - core flow number of %s on ONOS%s is %s" % ( device.dpid, controller.index, coreFlowNumOnos ) ) |
| 95 | checkResult = EventStates().FAIL |
| 96 | # Get flows for comparison |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 97 | flows = controller.CLI.flows() |
| 98 | try: |
| 99 | flows = json.loads( flows ) |
| 100 | except ( TypeError, ValueError ): |
| 101 | main.log.exception( "Flow Check - Object not as expected: {!r}".format( flows ) ) |
| 102 | return EventStates().FAIL |
| 103 | # Compare flow IDs in ONOS and Mininet |
| 104 | flowIDList = [] |
| 105 | for item in flows: |
| 106 | for flow in item[ "flows" ]: |
| 107 | flowIDList.append( hex( int( flow[ 'id' ] ) ) ) |
| 108 | main.log.info( "Flow Check - current flow number on ONOS%s: %s" % ( controller.index, len( flowIDList ) ) ) |
| 109 | switchList = [] |
| 110 | for device in main.devices: |
| 111 | switchList.append( device.name ) |
You Wang | c61aaa2 | 2019-02-01 15:49:48 -0800 | [diff] [blame^] | 112 | with main.networkLock: |
| 113 | flowCompareResult = main.Network.checkFlowId( switchList, flowIDList, debug=False ) |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 114 | if not flowCompareResult: |
| 115 | main.log.warn( "Flow Check - flows on ONOS%s do not match that in Mininet" % ( controller.index ) ) |
| 116 | checkResult = EventStates().FAIL |
| 117 | # Check flow state |
| 118 | flowState = controller.CLI.checkFlowsState( isPENDING=False ) |
| 119 | if not flowState: |
| 120 | main.log.warn( "Flow Check - not all flows are in ADDED state on ONOS%s" % ( controller.index ) ) |
| 121 | checkResult = EventStates().FAIL |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 122 | return checkResult |
| 123 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 124 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 125 | class TopoCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 126 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 127 | def __init__( self ): |
| 128 | CheckEvent.__init__( self ) |
| 129 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 130 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 131 | |
| 132 | def startCheckEvent( self, args=None ): |
| 133 | import json |
| 134 | checkResult = EventStates().PASS |
| 135 | upLinkNum = 0 |
| 136 | upDeviceNum = 0 |
You Wang | 469d4ac | 2019-01-31 12:33:26 -0800 | [diff] [blame] | 137 | upHostList = [] |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 138 | with main.variableLock: |
| 139 | for link in main.links: |
| 140 | if not link.isDown() and not link.isRemoved(): |
| 141 | upLinkNum += 1 |
| 142 | for device in main.devices: |
| 143 | if not device.isDown() and not device.isRemoved(): |
| 144 | upDeviceNum += 1 |
| 145 | for host in main.hosts: |
| 146 | if not host.isDown() and not host.isRemoved(): |
You Wang | 469d4ac | 2019-01-31 12:33:26 -0800 | [diff] [blame] | 147 | upHostList.append( host ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 148 | clusterNum = 1 |
You Wang | 469d4ac | 2019-01-31 12:33:26 -0800 | [diff] [blame] | 149 | # Verify host IPs |
| 150 | if hasattr( main, "expectedHosts" ): |
| 151 | for host in upHostList: |
You Wang | c61aaa2 | 2019-02-01 15:49:48 -0800 | [diff] [blame^] | 152 | ipResult = main.Network.verifyHostIp( hostList=[ host.name ], |
You Wang | 469d4ac | 2019-01-31 12:33:26 -0800 | [diff] [blame] | 153 | prefix=main.expectedHosts[ 'network' ][ host.name ], |
| 154 | update=False ) |
| 155 | if not ipResult: |
| 156 | checkResult = EventStates().FAIL |
| 157 | main.log.warn( "Topo Check - Failed to verify IP address on host %s" % ( host.name ) ) |
| 158 | ''' |
You Wang | c61aaa2 | 2019-02-01 15:49:48 -0800 | [diff] [blame^] | 159 | with main.networkLock: |
| 160 | graphDictMininet = main.Network.getGraphDict( useId=True, |
| 161 | excludeNodes=main.excludeNodes ) |
You Wang | 469d4ac | 2019-01-31 12:33:26 -0800 | [diff] [blame] | 162 | ''' |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 163 | for controller in main.controllers: |
| 164 | if controller.isUp(): |
| 165 | with controller.CLILock: |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 166 | ''' |
Flavio Castro | 82ee2f6 | 2016-06-07 15:04:12 -0700 | [diff] [blame] | 167 | topoState = controller.CLI.checkStatus( upDeviceNum, upLinkNum ) |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 168 | if not topoState: |
| 169 | main.log.warn( "Topo Check - link or device number discoverd by ONOS%s is incorrect" % ( controller.index ) ) |
| 170 | checkResult = EventStates().FAIL |
You Wang | 221db32 | 2016-06-03 15:45:52 -0700 | [diff] [blame] | 171 | # Compare ONOS and Mininet topologies |
| 172 | graphDictONOS = controller.CLI.getGraphDict() |
| 173 | compareResult = main.graph.compareGraphs( graphDictONOS, graphDictMininet ) |
| 174 | if not compareResult: |
| 175 | checkResult = EventStates().FAIL |
| 176 | main.log.warn( "Topo Check - ONOS and Mininet topologies do not match" ) |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 177 | ''' |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 178 | try: |
You Wang | 221db32 | 2016-06-03 15:45:52 -0700 | [diff] [blame] | 179 | # Check links |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 180 | links = controller.CLI.links() |
| 181 | links = json.loads( links ) |
| 182 | if not len( links ) == upLinkNum: |
| 183 | checkResult = EventStates().FAIL |
| 184 | main.log.warn( "Topo Check - link number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upLinkNum, len( links ) ) ) |
| 185 | # Check devices |
| 186 | devices = controller.CLI.devices() |
| 187 | devices = json.loads( devices ) |
| 188 | availableDeviceNum = 0 |
| 189 | for device in devices: |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 190 | if device[ 'available' ]: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 191 | availableDeviceNum += 1 |
| 192 | if not availableDeviceNum == upDeviceNum: |
| 193 | checkResult = EventStates().FAIL |
| 194 | main.log.warn( "Topo Check - device number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upDeviceNum, availableDeviceNum ) ) |
| 195 | # Check hosts |
| 196 | hosts = controller.CLI.hosts() |
| 197 | hosts = json.loads( hosts ) |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 198 | if hasattr( main, "expectedHosts" ): |
| 199 | hosts = [ host for host in hosts if host[ 'id' ] in main.expectedHosts[ 'onos' ].keys() ] |
You Wang | 469d4ac | 2019-01-31 12:33:26 -0800 | [diff] [blame] | 200 | if not len( hosts ) == len( upHostList ): |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 201 | # checkResult = EventStates().FAIL |
You Wang | 469d4ac | 2019-01-31 12:33:26 -0800 | [diff] [blame] | 202 | main.log.warn( "Topo Check - host number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, len( upHostList ), len( hosts ) ) ) |
| 203 | # Check Host IPs |
| 204 | if hasattr( main, "expectedHosts" ): |
| 205 | for host in upHostList: |
| 206 | ipResult = controller.CLI.verifyHostIp( hostList=[ host.id ], |
| 207 | prefix=main.expectedHosts[ 'onos' ][ host.id ] ) |
| 208 | if not ipResult: |
| 209 | checkResult = EventStates().FAIL |
| 210 | main.log.warn( "Topo Check - ONOS%s failed to verify IP address on host %s" % ( controller.index, host.id ) ) |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 211 | # Check clusters |
| 212 | clusters = controller.CLI.clusters() |
| 213 | clusters = json.loads( clusters ) |
| 214 | if not len( clusters ) == clusterNum: |
| 215 | checkResult = EventStates().FAIL |
| 216 | main.log.warn( "Topo Check - cluster number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, clusterNum, len( clusters ) ) ) |
| 217 | except ( TypeError, ValueError ): |
| 218 | main.log.exception( "Flow Check - Object not as expected" ) |
| 219 | return EventStates().FAIL |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 220 | return checkResult |
| 221 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 222 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 223 | class ONOSCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 224 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 225 | def __init__( self ): |
| 226 | CheckEvent.__init__( self ) |
| 227 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 228 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 229 | |
| 230 | def startCheckEvent( self, args=None ): |
| 231 | import json |
| 232 | checkResult = EventStates().PASS |
| 233 | topics = [] |
| 234 | # TODO: Other topics? |
| 235 | for i in range( 14 ): |
Jon Hall | 8dafdcc | 2016-09-16 10:21:25 -0700 | [diff] [blame] | 236 | topics.append( "work-partition-" + str( i ) ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 237 | dpidToAvailability = {} |
| 238 | dpidToMaster = {} |
| 239 | for device in main.devices: |
| 240 | if device.isDown() or device.isRemoved(): |
| 241 | dpidToAvailability[ device.dpid ] = False |
| 242 | else: |
| 243 | dpidToAvailability[ device.dpid ] = True |
| 244 | dpidToMaster[ device.dpid ] = 'unknown' |
| 245 | # Check mastership, leaders and node states on each controller node |
| 246 | for controller in main.controllers: |
| 247 | if controller.isUp(): |
| 248 | # Check mastership |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 249 | try: |
| 250 | with controller.CLILock: |
| 251 | roles = controller.CLI.roles() |
| 252 | roles = json.loads( roles ) |
| 253 | for device in roles: |
| 254 | dpid = device[ 'id' ] |
| 255 | if dpidToMaster[ dpid ] == 'unknown': |
| 256 | dpidToMaster[ dpid ] = device[ 'master' ] |
| 257 | elif dpidToMaster[ dpid ] != device[ 'master' ]: |
| 258 | checkResult = EventStates().FAIL |
| 259 | main.log.warn( "ONOS Check - Mastership of %s on ONOS%s is inconsistent with that on ONOS1" % ( dpid, controller.index ) ) |
| 260 | if dpidToAvailability[ dpid ] and device[ 'master' ] == "none": |
| 261 | checkResult = EventStates().FAIL |
| 262 | main.log.warn( "ONOS Check - Device %s has no master on ONOS%s" % ( dpid, controller.index ) ) |
| 263 | # Check leaders |
| 264 | with controller.CLILock: |
| 265 | leaders = controller.CLI.leaders() |
| 266 | leaders = json.loads( leaders ) |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 267 | ONOSTopics = [ j[ 'topic' ] for j in leaders ] |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 268 | for topic in topics: |
| 269 | if topic not in ONOSTopics: |
| 270 | checkResult = EventStates().FAIL |
| 271 | main.log.warn( "ONOS Check - Topic %s not in leaders on ONOS%s" % ( topic, controller.index ) ) |
| 272 | # Check node state |
| 273 | with controller.CLILock: |
| 274 | nodes = controller.CLI.nodes() |
| 275 | nodes = json.loads( nodes ) |
| 276 | ipToState = {} |
| 277 | for node in nodes: |
| 278 | ipToState[ node[ 'ip' ] ] = node[ 'state' ] |
| 279 | for c in main.controllers: |
| 280 | if c.isUp() and ipToState[ c.ip ] == 'READY': |
| 281 | pass |
You Wang | 817990a | 2019-01-22 15:07:25 -0800 | [diff] [blame] | 282 | elif not c.isUp() and c.ip not in ipToState.keys(): |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 283 | pass |
| 284 | else: |
| 285 | checkResult = EventStates().FAIL |
| 286 | main.log.warn( "ONOS Check - ONOS%s shows wrong node state: ONOS%s is %s but state is %s" % ( controller.index, c.index, c.status, ipToState[ c.ip ] ) ) |
| 287 | # TODO: check partitions? |
| 288 | except ( TypeError, ValueError ): |
| 289 | main.log.exception( "ONOS Check - Object not as expected" ) |
| 290 | return EventStates().FAIL |
You Wang | 817990a | 2019-01-22 15:07:25 -0800 | [diff] [blame] | 291 | except Exception as e: |
| 292 | main.log.exception( "ONOS Check - Uncaught Exception: {}".format( e ) ) |
| 293 | return EventStates().FAIL |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 294 | return checkResult |
| 295 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 296 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 297 | class TrafficCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 298 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 299 | def __init__( self ): |
| 300 | CheckEvent.__init__( self ) |
| 301 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 302 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 303 | |
| 304 | def startCheckEvent( self, args=None ): |
| 305 | checkResult = EventStates().PASS |
| 306 | pool = [] |
| 307 | wait = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingWait' ] ) |
| 308 | timeout = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingTimeout' ] ) |
| 309 | dstIPv4List = {} |
| 310 | dstIPv6List = {} |
| 311 | upHosts = [] |
| 312 | for host in main.hosts: |
| 313 | if host.isUp(): |
| 314 | upHosts.append( host ) |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 315 | import re |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 316 | for host in upHosts: |
| 317 | dstIPv4List[ host.index ] = [] |
| 318 | dstIPv6List[ host.index ] = [] |
| 319 | for correspondent in host.correspondents: |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 320 | if correspondent not in upHosts: |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 321 | continue |
| 322 | for ipAddress in correspondent.ipAddresses: |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 323 | if re.match( str( main.params[ 'TEST' ][ 'ipv6Regex' ] ), ipAddress ): |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 324 | dstIPv6List[ host.index ].append( ipAddress ) |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 325 | elif re.match( str( main.params[ 'TEST' ][ 'ipv4Regex' ] ), ipAddress ): |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 326 | dstIPv4List[ host.index ].append( ipAddress ) |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 327 | if dstIPv4List[ host.index ]: |
| 328 | main.log.debug( "Check ping from host {} to {}".format( host.name, dstIPv4List[ host.index ] ) ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 329 | thread = main.Thread( target=host.handle.pingHostSetAlternative, |
| 330 | threadID=main.threadID, |
| 331 | name="pingHostSetAlternative", |
| 332 | args=[ dstIPv4List[ host.index ], 1 ] ) |
| 333 | pool.append( thread ) |
| 334 | thread.start() |
| 335 | with main.variableLock: |
| 336 | main.threadID += 1 |
| 337 | for thread in pool: |
| 338 | thread.join( 10 ) |
| 339 | if not thread.result: |
| 340 | checkResult = EventStates().FAIL |
| 341 | main.log.warn( "Traffic Check - ping failed" ) |
| 342 | |
| 343 | if not main.enableIPv6: |
| 344 | return checkResult |
| 345 | # Check ipv6 ping |
| 346 | for host in upHosts: |
You Wang | 9fc5ce4 | 2019-01-23 15:10:08 -0800 | [diff] [blame] | 347 | if dstIPv6List[ host.index ]: |
| 348 | main.log.debug( "Check ping from host {} to {}".format( host.name, dstIPv6List[ host.index ] ) ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 349 | thread = main.Thread( target=host.handle.pingHostSetAlternative, |
| 350 | threadID=main.threadID, |
| 351 | name="pingHostSetAlternative", |
| 352 | args=[ dstIPv6List[ host.index ], 1, True ] ) |
| 353 | pool.append( thread ) |
| 354 | thread.start() |
| 355 | with main.variableLock: |
| 356 | main.threadID += 1 |
| 357 | for thread in pool: |
| 358 | thread.join( 10 ) |
| 359 | if not thread.result: |
| 360 | checkResult = EventStates().FAIL |
| 361 | main.log.warn( "Traffic Check - ping6 failed" ) |
| 362 | return checkResult |
Devin Lim | f082218 | 2017-09-12 14:52:57 -0700 | [diff] [blame] | 363 | |
| 364 | class RaftLogSizeCheck( CheckEvent ): |
| 365 | |
| 366 | def __init__( self ): |
| 367 | CheckEvent.__init__( self ) |
| 368 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 369 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 370 | |
| 371 | def startCheckEvent( self, args=None ): |
| 372 | checkResult = EventStates().PASS |
| 373 | main.log.info( "Starting checking Raft Log size" ) |
| 374 | if not main.Cluster.checkPartitionSize(): |
| 375 | checkResult = EventStates().FAIL |
| 376 | main.log.warn( "Raft Log Size Check - Raft log grew too big" ) |
| 377 | |
| 378 | return checkResult |