You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 1 | """ |
| 2 | This file contains classes for CHOTestMonkey that are related to check event |
| 3 | Author: you@onlab.us |
| 4 | """ |
| 5 | from tests.CHOTestMonkey.dependencies.events.Event import EventType, EventStates, Event |
| 6 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 7 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 8 | class CheckEvent( Event ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 9 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 10 | def __init__( self ): |
| 11 | Event.__init__( self ) |
| 12 | |
| 13 | def startCheckEvent( self ): |
| 14 | return EventStates().PASS |
| 15 | |
| 16 | def startEvent( self, args ): |
| 17 | with self.eventLock: |
You Wang | 5216320 | 2016-07-14 16:37:15 -0700 | [diff] [blame] | 18 | main.log.info( "Event recorded: {} {}".format( self.typeIndex, self.typeString ) ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 19 | result = self.startCheckEvent() |
| 20 | return result |
| 21 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 22 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 23 | class IntentCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 24 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 25 | def __init__( self ): |
| 26 | CheckEvent.__init__( self ) |
| 27 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 28 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 29 | |
| 30 | def startCheckEvent( self, args=None ): |
| 31 | checkResult = EventStates().PASS |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 32 | intentDict = {} |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 33 | for intent in main.intents: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 34 | intentDict[ intent.id ] = intent.expectedState |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 35 | for controller in main.controllers: |
| 36 | if controller.isUp(): |
| 37 | with controller.CLILock: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 38 | intentState = controller.CLI.compareIntent( intentDict ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 39 | if not intentState: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 40 | main.log.warn( "Intent Check - not all intent ids and states match that on ONOS%s" % ( controller.index ) ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 41 | checkResult = EventStates().FAIL |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 42 | return checkResult |
| 43 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 44 | |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 45 | class FlowCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 46 | |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 47 | def __init__( self ): |
| 48 | CheckEvent.__init__( self ) |
| 49 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 50 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 51 | |
| 52 | def startCheckEvent( self, args=None ): |
| 53 | import json |
| 54 | checkResult = EventStates().PASS |
You Wang | 3c27625 | 2016-09-21 15:21:36 -0700 | [diff] [blame] | 55 | if main.enableIPv6: |
You Wang | 62e1cf6 | 2016-09-22 17:13:03 -0700 | [diff] [blame] | 56 | coreFlowNum = int( main.params[ 'EVENT' ][ 'FlowCheck' ][ 'coreFlowNum6' ] ) |
You Wang | 3c27625 | 2016-09-21 15:21:36 -0700 | [diff] [blame] | 57 | else: |
You Wang | 62e1cf6 | 2016-09-22 17:13:03 -0700 | [diff] [blame] | 58 | coreFlowNum = int( main.params[ 'EVENT' ][ 'FlowCheck' ][ 'coreFlowNum' ] ) |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 59 | for controller in main.controllers: |
| 60 | if controller.isUp(): |
| 61 | with controller.CLILock: |
You Wang | 3c27625 | 2016-09-21 15:21:36 -0700 | [diff] [blame] | 62 | # Check core flow number |
| 63 | for device in main.devices: |
| 64 | if device.isRemoved(): |
| 65 | continue |
| 66 | coreFlowNumOnos = controller.CLI.flowAddedCount( device.dpid, core=True ) |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 67 | if coreFlowNumOnos is None: |
You Wang | 3c27625 | 2016-09-21 15:21:36 -0700 | [diff] [blame] | 68 | main.log.warn( "Flow Check - error when trying to get flow number of %s on ONOS%s" % ( device.dpid, controller.index ) ) |
| 69 | checkResult = EventStates().FAIL |
| 70 | else: |
| 71 | coreFlowNumOnos = int( coreFlowNumOnos ) |
| 72 | if coreFlowNumOnos != coreFlowNum: |
| 73 | main.log.warn( "Flow Check - core flow number of %s on ONOS%s is %s" % ( device.dpid, controller.index, coreFlowNumOnos ) ) |
| 74 | checkResult = EventStates().FAIL |
| 75 | # Get flows for comparison |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 76 | flows = controller.CLI.flows() |
| 77 | try: |
| 78 | flows = json.loads( flows ) |
| 79 | except ( TypeError, ValueError ): |
| 80 | main.log.exception( "Flow Check - Object not as expected: {!r}".format( flows ) ) |
| 81 | return EventStates().FAIL |
| 82 | # Compare flow IDs in ONOS and Mininet |
| 83 | flowIDList = [] |
| 84 | for item in flows: |
| 85 | for flow in item[ "flows" ]: |
| 86 | flowIDList.append( hex( int( flow[ 'id' ] ) ) ) |
| 87 | main.log.info( "Flow Check - current flow number on ONOS%s: %s" % ( controller.index, len( flowIDList ) ) ) |
| 88 | switchList = [] |
| 89 | for device in main.devices: |
| 90 | switchList.append( device.name ) |
| 91 | with main.mininetLock: |
| 92 | flowCompareResult = main.Mininet1.checkFlowId( switchList, flowIDList, debug=False ) |
| 93 | if not flowCompareResult: |
| 94 | main.log.warn( "Flow Check - flows on ONOS%s do not match that in Mininet" % ( controller.index ) ) |
| 95 | checkResult = EventStates().FAIL |
| 96 | # Check flow state |
| 97 | flowState = controller.CLI.checkFlowsState( isPENDING=False ) |
| 98 | if not flowState: |
| 99 | main.log.warn( "Flow Check - not all flows are in ADDED state on ONOS%s" % ( controller.index ) ) |
| 100 | checkResult = EventStates().FAIL |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 101 | return checkResult |
| 102 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 103 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 104 | class TopoCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 105 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 106 | def __init__( self ): |
| 107 | CheckEvent.__init__( self ) |
| 108 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 109 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 110 | |
| 111 | def startCheckEvent( self, args=None ): |
| 112 | import json |
| 113 | checkResult = EventStates().PASS |
| 114 | upLinkNum = 0 |
| 115 | upDeviceNum = 0 |
| 116 | upHostNum = 0 |
| 117 | with main.variableLock: |
| 118 | for link in main.links: |
| 119 | if not link.isDown() and not link.isRemoved(): |
| 120 | upLinkNum += 1 |
| 121 | for device in main.devices: |
| 122 | if not device.isDown() and not device.isRemoved(): |
| 123 | upDeviceNum += 1 |
| 124 | for host in main.hosts: |
| 125 | if not host.isDown() and not host.isRemoved(): |
| 126 | upHostNum += 1 |
| 127 | clusterNum = 1 |
You Wang | 221db32 | 2016-06-03 15:45:52 -0700 | [diff] [blame] | 128 | with main.mininetLock: |
| 129 | graphDictMininet = main.Mininet1.getGraphDict( useId=True ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 130 | for controller in main.controllers: |
| 131 | if controller.isUp(): |
| 132 | with controller.CLILock: |
Flavio Castro | 82ee2f6 | 2016-06-07 15:04:12 -0700 | [diff] [blame] | 133 | topoState = controller.CLI.checkStatus( upDeviceNum, upLinkNum ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 134 | #if not topoState: |
| 135 | # main.log.warn( "Topo Check - link or device number discoverd by ONOS%s is incorrect" % ( controller.index ) ) |
| 136 | # checkResult = EventStates().FAIL |
You Wang | 221db32 | 2016-06-03 15:45:52 -0700 | [diff] [blame] | 137 | # Compare ONOS and Mininet topologies |
| 138 | graphDictONOS = controller.CLI.getGraphDict() |
| 139 | compareResult = main.graph.compareGraphs( graphDictONOS, graphDictMininet ) |
| 140 | if not compareResult: |
| 141 | checkResult = EventStates().FAIL |
| 142 | main.log.warn( "Topo Check - ONOS and Mininet topologies do not match" ) |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 143 | try: |
You Wang | 221db32 | 2016-06-03 15:45:52 -0700 | [diff] [blame] | 144 | # Check links |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 145 | links = controller.CLI.links() |
| 146 | links = json.loads( links ) |
| 147 | if not len( links ) == upLinkNum: |
| 148 | checkResult = EventStates().FAIL |
| 149 | main.log.warn( "Topo Check - link number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upLinkNum, len( links ) ) ) |
| 150 | # Check devices |
| 151 | devices = controller.CLI.devices() |
| 152 | devices = json.loads( devices ) |
| 153 | availableDeviceNum = 0 |
| 154 | for device in devices: |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 155 | if device[ 'available' ]: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 156 | availableDeviceNum += 1 |
| 157 | if not availableDeviceNum == upDeviceNum: |
| 158 | checkResult = EventStates().FAIL |
| 159 | main.log.warn( "Topo Check - device number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upDeviceNum, availableDeviceNum ) ) |
| 160 | # Check hosts |
| 161 | hosts = controller.CLI.hosts() |
| 162 | hosts = json.loads( hosts ) |
| 163 | if not len( hosts ) == upHostNum: |
| 164 | checkResult = EventStates().FAIL |
| 165 | main.log.warn( "Topo Check - host number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upHostNum, len( hosts ) ) ) |
| 166 | # Check clusters |
| 167 | clusters = controller.CLI.clusters() |
| 168 | clusters = json.loads( clusters ) |
| 169 | if not len( clusters ) == clusterNum: |
| 170 | checkResult = EventStates().FAIL |
| 171 | main.log.warn( "Topo Check - cluster number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, clusterNum, len( clusters ) ) ) |
| 172 | except ( TypeError, ValueError ): |
| 173 | main.log.exception( "Flow Check - Object not as expected" ) |
| 174 | return EventStates().FAIL |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 175 | return checkResult |
| 176 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 177 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 178 | class ONOSCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 179 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 180 | def __init__( self ): |
| 181 | CheckEvent.__init__( self ) |
| 182 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 183 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 184 | |
| 185 | def startCheckEvent( self, args=None ): |
| 186 | import json |
| 187 | checkResult = EventStates().PASS |
| 188 | topics = [] |
| 189 | # TODO: Other topics? |
| 190 | for i in range( 14 ): |
Jon Hall | 8dafdcc | 2016-09-16 10:21:25 -0700 | [diff] [blame] | 191 | topics.append( "work-partition-" + str( i ) ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 192 | dpidToAvailability = {} |
| 193 | dpidToMaster = {} |
| 194 | for device in main.devices: |
| 195 | if device.isDown() or device.isRemoved(): |
| 196 | dpidToAvailability[ device.dpid ] = False |
| 197 | else: |
| 198 | dpidToAvailability[ device.dpid ] = True |
| 199 | dpidToMaster[ device.dpid ] = 'unknown' |
| 200 | # Check mastership, leaders and node states on each controller node |
| 201 | for controller in main.controllers: |
| 202 | if controller.isUp(): |
| 203 | # Check mastership |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 204 | try: |
| 205 | with controller.CLILock: |
| 206 | roles = controller.CLI.roles() |
| 207 | roles = json.loads( roles ) |
| 208 | for device in roles: |
| 209 | dpid = device[ 'id' ] |
| 210 | if dpidToMaster[ dpid ] == 'unknown': |
| 211 | dpidToMaster[ dpid ] = device[ 'master' ] |
| 212 | elif dpidToMaster[ dpid ] != device[ 'master' ]: |
| 213 | checkResult = EventStates().FAIL |
| 214 | main.log.warn( "ONOS Check - Mastership of %s on ONOS%s is inconsistent with that on ONOS1" % ( dpid, controller.index ) ) |
| 215 | if dpidToAvailability[ dpid ] and device[ 'master' ] == "none": |
| 216 | checkResult = EventStates().FAIL |
| 217 | main.log.warn( "ONOS Check - Device %s has no master on ONOS%s" % ( dpid, controller.index ) ) |
| 218 | # Check leaders |
| 219 | with controller.CLILock: |
| 220 | leaders = controller.CLI.leaders() |
| 221 | leaders = json.loads( leaders ) |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 222 | ONOSTopics = [ j[ 'topic' ] for j in leaders ] |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 223 | for topic in topics: |
| 224 | if topic not in ONOSTopics: |
| 225 | checkResult = EventStates().FAIL |
| 226 | main.log.warn( "ONOS Check - Topic %s not in leaders on ONOS%s" % ( topic, controller.index ) ) |
| 227 | # Check node state |
| 228 | with controller.CLILock: |
| 229 | nodes = controller.CLI.nodes() |
| 230 | nodes = json.loads( nodes ) |
| 231 | ipToState = {} |
| 232 | for node in nodes: |
| 233 | ipToState[ node[ 'ip' ] ] = node[ 'state' ] |
| 234 | for c in main.controllers: |
| 235 | if c.isUp() and ipToState[ c.ip ] == 'READY': |
| 236 | pass |
| 237 | elif not c.isUp() and ipToState[ c.ip ] == 'INACTIVE': |
| 238 | pass |
| 239 | else: |
| 240 | checkResult = EventStates().FAIL |
| 241 | main.log.warn( "ONOS Check - ONOS%s shows wrong node state: ONOS%s is %s but state is %s" % ( controller.index, c.index, c.status, ipToState[ c.ip ] ) ) |
| 242 | # TODO: check partitions? |
| 243 | except ( TypeError, ValueError ): |
| 244 | main.log.exception( "ONOS Check - Object not as expected" ) |
| 245 | return EventStates().FAIL |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 246 | return checkResult |
| 247 | |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 248 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 249 | class TrafficCheck( CheckEvent ): |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 250 | |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 251 | def __init__( self ): |
| 252 | CheckEvent.__init__( self ) |
| 253 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 254 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 255 | |
| 256 | def startCheckEvent( self, args=None ): |
| 257 | checkResult = EventStates().PASS |
| 258 | pool = [] |
| 259 | wait = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingWait' ] ) |
| 260 | timeout = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingTimeout' ] ) |
| 261 | dstIPv4List = {} |
| 262 | dstIPv6List = {} |
| 263 | upHosts = [] |
| 264 | for host in main.hosts: |
| 265 | if host.isUp(): |
| 266 | upHosts.append( host ) |
| 267 | for host in upHosts: |
| 268 | dstIPv4List[ host.index ] = [] |
| 269 | dstIPv6List[ host.index ] = [] |
| 270 | for correspondent in host.correspondents: |
Jon Hall | 2bb3e21 | 2017-05-24 17:07:25 -0700 | [diff] [blame] | 271 | if correspondent not in upHosts: |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 272 | continue |
| 273 | for ipAddress in correspondent.ipAddresses: |
| 274 | if ipAddress.startswith( str( main.params[ 'TEST' ][ 'ipv6Prefix' ] ) ): |
| 275 | dstIPv6List[ host.index ].append( ipAddress ) |
| 276 | elif ipAddress.startswith( str( main.params[ 'TEST' ][ 'ipv4Prefix' ] ) ): |
| 277 | dstIPv4List[ host.index ].append( ipAddress ) |
| 278 | thread = main.Thread( target=host.handle.pingHostSetAlternative, |
| 279 | threadID=main.threadID, |
| 280 | name="pingHostSetAlternative", |
| 281 | args=[ dstIPv4List[ host.index ], 1 ] ) |
| 282 | pool.append( thread ) |
| 283 | thread.start() |
| 284 | with main.variableLock: |
| 285 | main.threadID += 1 |
| 286 | for thread in pool: |
| 287 | thread.join( 10 ) |
| 288 | if not thread.result: |
| 289 | checkResult = EventStates().FAIL |
| 290 | main.log.warn( "Traffic Check - ping failed" ) |
| 291 | |
| 292 | if not main.enableIPv6: |
| 293 | return checkResult |
| 294 | # Check ipv6 ping |
| 295 | for host in upHosts: |
| 296 | thread = main.Thread( target=host.handle.pingHostSetAlternative, |
| 297 | threadID=main.threadID, |
| 298 | name="pingHostSetAlternative", |
| 299 | args=[ dstIPv6List[ host.index ], 1, True ] ) |
| 300 | pool.append( thread ) |
| 301 | thread.start() |
| 302 | with main.variableLock: |
| 303 | main.threadID += 1 |
| 304 | for thread in pool: |
| 305 | thread.join( 10 ) |
| 306 | if not thread.result: |
| 307 | checkResult = EventStates().FAIL |
| 308 | main.log.warn( "Traffic Check - ping6 failed" ) |
| 309 | return checkResult |