You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 1 | """ |
| 2 | This file contains classes for CHOTestMonkey that are related to check event |
| 3 | Author: you@onlab.us |
| 4 | """ |
| 5 | from tests.CHOTestMonkey.dependencies.events.Event import EventType, EventStates, Event |
| 6 | |
| 7 | class CheckEvent( Event ): |
| 8 | def __init__( self ): |
| 9 | Event.__init__( self ) |
| 10 | |
| 11 | def startCheckEvent( self ): |
| 12 | return EventStates().PASS |
| 13 | |
| 14 | def startEvent( self, args ): |
| 15 | with self.eventLock: |
| 16 | main.log.info( "%s - starting event" % ( self.typeString ) ) |
| 17 | result = self.startCheckEvent() |
| 18 | return result |
| 19 | |
| 20 | class IntentCheck( CheckEvent ): |
| 21 | def __init__( self ): |
| 22 | CheckEvent.__init__( self ) |
| 23 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 24 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 25 | |
| 26 | def startCheckEvent( self, args=None ): |
| 27 | checkResult = EventStates().PASS |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 28 | intentDict = {} |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 29 | for intent in main.intents: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 30 | intentDict[ intent.id ] = intent.expectedState |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 31 | for controller in main.controllers: |
| 32 | if controller.isUp(): |
| 33 | with controller.CLILock: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 34 | intentState = controller.CLI.compareIntent( intentDict ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 35 | if not intentState: |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 36 | main.log.warn( "Intent Check - not all intent ids and states match that on ONOS%s" % ( controller.index ) ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 37 | checkResult = EventStates().FAIL |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 38 | return checkResult |
| 39 | |
| 40 | class FlowCheck( CheckEvent ): |
| 41 | def __init__( self ): |
| 42 | CheckEvent.__init__( self ) |
| 43 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 44 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 45 | |
| 46 | def startCheckEvent( self, args=None ): |
| 47 | import json |
| 48 | checkResult = EventStates().PASS |
| 49 | for controller in main.controllers: |
| 50 | if controller.isUp(): |
| 51 | with controller.CLILock: |
| 52 | flows = controller.CLI.flows() |
| 53 | try: |
| 54 | flows = json.loads( flows ) |
| 55 | except ( TypeError, ValueError ): |
| 56 | main.log.exception( "Flow Check - Object not as expected: {!r}".format( flows ) ) |
| 57 | return EventStates().FAIL |
| 58 | # Compare flow IDs in ONOS and Mininet |
| 59 | flowIDList = [] |
| 60 | for item in flows: |
| 61 | for flow in item[ "flows" ]: |
| 62 | flowIDList.append( hex( int( flow[ 'id' ] ) ) ) |
| 63 | main.log.info( "Flow Check - current flow number on ONOS%s: %s" % ( controller.index, len( flowIDList ) ) ) |
| 64 | switchList = [] |
| 65 | for device in main.devices: |
| 66 | switchList.append( device.name ) |
| 67 | with main.mininetLock: |
| 68 | flowCompareResult = main.Mininet1.checkFlowId( switchList, flowIDList, debug=False ) |
| 69 | if not flowCompareResult: |
| 70 | main.log.warn( "Flow Check - flows on ONOS%s do not match that in Mininet" % ( controller.index ) ) |
| 71 | checkResult = EventStates().FAIL |
| 72 | # Check flow state |
| 73 | flowState = controller.CLI.checkFlowsState( isPENDING=False ) |
| 74 | if not flowState: |
| 75 | main.log.warn( "Flow Check - not all flows are in ADDED state on ONOS%s" % ( controller.index ) ) |
| 76 | checkResult = EventStates().FAIL |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 77 | return checkResult |
| 78 | |
| 79 | class TopoCheck( CheckEvent ): |
| 80 | def __init__( self ): |
| 81 | CheckEvent.__init__( self ) |
| 82 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 83 | self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 84 | |
| 85 | def startCheckEvent( self, args=None ): |
| 86 | import json |
| 87 | checkResult = EventStates().PASS |
| 88 | upLinkNum = 0 |
| 89 | upDeviceNum = 0 |
| 90 | upHostNum = 0 |
| 91 | with main.variableLock: |
| 92 | for link in main.links: |
| 93 | if not link.isDown() and not link.isRemoved(): |
| 94 | upLinkNum += 1 |
| 95 | for device in main.devices: |
| 96 | if not device.isDown() and not device.isRemoved(): |
| 97 | upDeviceNum += 1 |
| 98 | for host in main.hosts: |
| 99 | if not host.isDown() and not host.isRemoved(): |
| 100 | upHostNum += 1 |
| 101 | clusterNum = 1 |
You Wang | 221db32 | 2016-06-03 15:45:52 -0700 | [diff] [blame] | 102 | with main.mininetLock: |
| 103 | graphDictMininet = main.Mininet1.getGraphDict( useId=True ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 104 | for controller in main.controllers: |
| 105 | if controller.isUp(): |
| 106 | with controller.CLILock: |
Flavio Castro | 82ee2f6 | 2016-06-07 15:04:12 -0700 | [diff] [blame] | 107 | topoState = controller.CLI.checkStatus( upDeviceNum, upLinkNum ) |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 108 | #if not topoState: |
| 109 | # main.log.warn( "Topo Check - link or device number discoverd by ONOS%s is incorrect" % ( controller.index ) ) |
| 110 | # checkResult = EventStates().FAIL |
You Wang | 221db32 | 2016-06-03 15:45:52 -0700 | [diff] [blame] | 111 | # Compare ONOS and Mininet topologies |
| 112 | graphDictONOS = controller.CLI.getGraphDict() |
| 113 | compareResult = main.graph.compareGraphs( graphDictONOS, graphDictMininet ) |
| 114 | if not compareResult: |
| 115 | checkResult = EventStates().FAIL |
| 116 | main.log.warn( "Topo Check - ONOS and Mininet topologies do not match" ) |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 117 | try: |
You Wang | 221db32 | 2016-06-03 15:45:52 -0700 | [diff] [blame] | 118 | # Check links |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 119 | links = controller.CLI.links() |
| 120 | links = json.loads( links ) |
| 121 | if not len( links ) == upLinkNum: |
| 122 | checkResult = EventStates().FAIL |
| 123 | main.log.warn( "Topo Check - link number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upLinkNum, len( links ) ) ) |
| 124 | # Check devices |
| 125 | devices = controller.CLI.devices() |
| 126 | devices = json.loads( devices ) |
| 127 | availableDeviceNum = 0 |
| 128 | for device in devices: |
| 129 | if device[ 'available' ] == True: |
| 130 | availableDeviceNum += 1 |
| 131 | if not availableDeviceNum == upDeviceNum: |
| 132 | checkResult = EventStates().FAIL |
| 133 | main.log.warn( "Topo Check - device number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upDeviceNum, availableDeviceNum ) ) |
| 134 | # Check hosts |
| 135 | hosts = controller.CLI.hosts() |
| 136 | hosts = json.loads( hosts ) |
| 137 | if not len( hosts ) == upHostNum: |
| 138 | checkResult = EventStates().FAIL |
| 139 | main.log.warn( "Topo Check - host number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upHostNum, len( hosts ) ) ) |
| 140 | # Check clusters |
| 141 | clusters = controller.CLI.clusters() |
| 142 | clusters = json.loads( clusters ) |
| 143 | if not len( clusters ) == clusterNum: |
| 144 | checkResult = EventStates().FAIL |
| 145 | main.log.warn( "Topo Check - cluster number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, clusterNum, len( clusters ) ) ) |
| 146 | except ( TypeError, ValueError ): |
| 147 | main.log.exception( "Flow Check - Object not as expected" ) |
| 148 | return EventStates().FAIL |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 149 | return checkResult |
| 150 | |
| 151 | class ONOSCheck( CheckEvent ): |
| 152 | def __init__( self ): |
| 153 | CheckEvent.__init__( self ) |
| 154 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 155 | self.typeIndex= int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 156 | |
| 157 | def startCheckEvent( self, args=None ): |
| 158 | import json |
| 159 | checkResult = EventStates().PASS |
| 160 | topics = [] |
| 161 | # TODO: Other topics? |
| 162 | for i in range( 14 ): |
| 163 | topics.append( "intent-partition-" + str( i ) ) |
| 164 | dpidToAvailability = {} |
| 165 | dpidToMaster = {} |
| 166 | for device in main.devices: |
| 167 | if device.isDown() or device.isRemoved(): |
| 168 | dpidToAvailability[ device.dpid ] = False |
| 169 | else: |
| 170 | dpidToAvailability[ device.dpid ] = True |
| 171 | dpidToMaster[ device.dpid ] = 'unknown' |
| 172 | # Check mastership, leaders and node states on each controller node |
| 173 | for controller in main.controllers: |
| 174 | if controller.isUp(): |
| 175 | # Check mastership |
You Wang | 58aa11e | 2016-05-17 10:35:44 -0700 | [diff] [blame] | 176 | try: |
| 177 | with controller.CLILock: |
| 178 | roles = controller.CLI.roles() |
| 179 | roles = json.loads( roles ) |
| 180 | for device in roles: |
| 181 | dpid = device[ 'id' ] |
| 182 | if dpidToMaster[ dpid ] == 'unknown': |
| 183 | dpidToMaster[ dpid ] = device[ 'master' ] |
| 184 | elif dpidToMaster[ dpid ] != device[ 'master' ]: |
| 185 | checkResult = EventStates().FAIL |
| 186 | main.log.warn( "ONOS Check - Mastership of %s on ONOS%s is inconsistent with that on ONOS1" % ( dpid, controller.index ) ) |
| 187 | if dpidToAvailability[ dpid ] and device[ 'master' ] == "none": |
| 188 | checkResult = EventStates().FAIL |
| 189 | main.log.warn( "ONOS Check - Device %s has no master on ONOS%s" % ( dpid, controller.index ) ) |
| 190 | # Check leaders |
| 191 | with controller.CLILock: |
| 192 | leaders = controller.CLI.leaders() |
| 193 | leaders = json.loads( leaders ) |
| 194 | ONOSTopics = [ j['topic'] for j in leaders ] |
| 195 | for topic in topics: |
| 196 | if topic not in ONOSTopics: |
| 197 | checkResult = EventStates().FAIL |
| 198 | main.log.warn( "ONOS Check - Topic %s not in leaders on ONOS%s" % ( topic, controller.index ) ) |
| 199 | # Check node state |
| 200 | with controller.CLILock: |
| 201 | nodes = controller.CLI.nodes() |
| 202 | nodes = json.loads( nodes ) |
| 203 | ipToState = {} |
| 204 | for node in nodes: |
| 205 | ipToState[ node[ 'ip' ] ] = node[ 'state' ] |
| 206 | for c in main.controllers: |
| 207 | if c.isUp() and ipToState[ c.ip ] == 'READY': |
| 208 | pass |
| 209 | elif not c.isUp() and ipToState[ c.ip ] == 'INACTIVE': |
| 210 | pass |
| 211 | else: |
| 212 | checkResult = EventStates().FAIL |
| 213 | main.log.warn( "ONOS Check - ONOS%s shows wrong node state: ONOS%s is %s but state is %s" % ( controller.index, c.index, c.status, ipToState[ c.ip ] ) ) |
| 214 | # TODO: check partitions? |
| 215 | except ( TypeError, ValueError ): |
| 216 | main.log.exception( "ONOS Check - Object not as expected" ) |
| 217 | return EventStates().FAIL |
You Wang | db927a5 | 2016-02-26 11:03:28 -0800 | [diff] [blame] | 218 | return checkResult |
| 219 | |
| 220 | class TrafficCheck( CheckEvent ): |
| 221 | def __init__( self ): |
| 222 | CheckEvent.__init__( self ) |
| 223 | self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ] |
| 224 | self.typeIndex= int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] ) |
| 225 | |
| 226 | def startCheckEvent( self, args=None ): |
| 227 | checkResult = EventStates().PASS |
| 228 | pool = [] |
| 229 | wait = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingWait' ] ) |
| 230 | timeout = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingTimeout' ] ) |
| 231 | dstIPv4List = {} |
| 232 | dstIPv6List = {} |
| 233 | upHosts = [] |
| 234 | for host in main.hosts: |
| 235 | if host.isUp(): |
| 236 | upHosts.append( host ) |
| 237 | for host in upHosts: |
| 238 | dstIPv4List[ host.index ] = [] |
| 239 | dstIPv6List[ host.index ] = [] |
| 240 | for correspondent in host.correspondents: |
| 241 | if not correspondent in upHosts: |
| 242 | continue |
| 243 | for ipAddress in correspondent.ipAddresses: |
| 244 | if ipAddress.startswith( str( main.params[ 'TEST' ][ 'ipv6Prefix' ] ) ): |
| 245 | dstIPv6List[ host.index ].append( ipAddress ) |
| 246 | elif ipAddress.startswith( str( main.params[ 'TEST' ][ 'ipv4Prefix' ] ) ): |
| 247 | dstIPv4List[ host.index ].append( ipAddress ) |
| 248 | thread = main.Thread( target=host.handle.pingHostSetAlternative, |
| 249 | threadID=main.threadID, |
| 250 | name="pingHostSetAlternative", |
| 251 | args=[ dstIPv4List[ host.index ], 1 ] ) |
| 252 | pool.append( thread ) |
| 253 | thread.start() |
| 254 | with main.variableLock: |
| 255 | main.threadID += 1 |
| 256 | for thread in pool: |
| 257 | thread.join( 10 ) |
| 258 | if not thread.result: |
| 259 | checkResult = EventStates().FAIL |
| 260 | main.log.warn( "Traffic Check - ping failed" ) |
| 261 | |
| 262 | if not main.enableIPv6: |
| 263 | return checkResult |
| 264 | # Check ipv6 ping |
| 265 | for host in upHosts: |
| 266 | thread = main.Thread( target=host.handle.pingHostSetAlternative, |
| 267 | threadID=main.threadID, |
| 268 | name="pingHostSetAlternative", |
| 269 | args=[ dstIPv6List[ host.index ], 1, True ] ) |
| 270 | pool.append( thread ) |
| 271 | thread.start() |
| 272 | with main.variableLock: |
| 273 | main.threadID += 1 |
| 274 | for thread in pool: |
| 275 | thread.join( 10 ) |
| 276 | if not thread.result: |
| 277 | checkResult = EventStates().FAIL |
| 278 | main.log.warn( "Traffic Check - ping6 failed" ) |
| 279 | return checkResult |
| 280 | |