Framework of the new CHOtest
Change-Id: Ie5b58bfa2ed487386443692cbea0d469d7419c24
diff --git a/TestON/tests/CHOTestMonkey/dependencies/events/CheckEvent.py b/TestON/tests/CHOTestMonkey/dependencies/events/CheckEvent.py
new file mode 100644
index 0000000..e48f674
--- /dev/null
+++ b/TestON/tests/CHOTestMonkey/dependencies/events/CheckEvent.py
@@ -0,0 +1,235 @@
+"""
+This file contains classes for CHOTestMonkey that are related to check event
+Author: you@onlab.us
+"""
+from tests.CHOTestMonkey.dependencies.events.Event import EventType, EventStates, Event
+
+class CheckEvent( Event ):
+ def __init__( self ):
+ Event.__init__( self )
+
+ def startCheckEvent( self ):
+ return EventStates().PASS
+
+ def startEvent( self, args ):
+ with self.eventLock:
+ main.log.info( "%s - starting event" % ( self.typeString ) )
+ result = self.startCheckEvent()
+ return result
+
+class IntentCheck( CheckEvent ):
+ def __init__( self ):
+ CheckEvent.__init__( self )
+ self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
+ self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
+
+ def startCheckEvent( self, args=None ):
+ checkResult = EventStates().PASS
+ intentIDs = []
+ for intent in main.intents:
+ if intent.isHostIntent():
+ deviceA = intent.hostA.device
+ deviceB = intent.hostB.device
+ elif intent.isPointIntent():
+ deviceA = intent.deviceA
+ deviceB = intent.deviceB
+ # Exclude the intents that are to or from removed devices/hosts
+ if not deviceA.isRemoved() and not deviceB.isRemoved():
+ intentIDs.append( intent.id )
+ for controller in main.controllers:
+ if controller.isUp():
+ with controller.CLILock:
+ intentState = controller.CLI.checkIntentState( intentsId=intentIDs )
+ if not intentState:
+ main.log.warn( "Intent Check - Not all intents are in INSTALLED state on ONOS%s" % ( controller.index ) )
+ checkResult = EventStates().FAIL
+ #TODO: check flows?
+ return checkResult
+
+class TopoCheck( CheckEvent ):
+ def __init__( self ):
+ CheckEvent.__init__( self )
+ self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
+ self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
+
+ def startCheckEvent( self, args=None ):
+ import json
+ checkResult = EventStates().PASS
+ upLinkNum = 0
+ upDeviceNum = 0
+ upHostNum = 0
+ with main.variableLock:
+ for link in main.links:
+ if not link.isDown() and not link.isRemoved():
+ upLinkNum += 1
+ for device in main.devices:
+ if not device.isDown() and not device.isRemoved():
+ upDeviceNum += 1
+ for host in main.hosts:
+ if not host.isDown() and not host.isRemoved():
+ upHostNum += 1
+ clusterNum = 1
+ for controller in main.controllers:
+ if controller.isUp():
+ with controller.CLILock:
+ topologyOutput = controller.CLI.topology()
+ topoState = controller.CLI.checkStatus( topologyOutput, upDeviceNum, upLinkNum )
+ #if not topoState:
+ # main.log.warn( "Topo Check - link or device number discoverd by ONOS%s is incorrect" % ( controller.index ) )
+ # checkResult = EventStates().FAIL
+ # Check links
+ links = controller.CLI.links()
+ links = json.loads( links )
+ if not len( links ) == upLinkNum:
+ checkResult = EventStates().FAIL
+ main.log.warn( "Topo Check - link number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upLinkNum, len( links ) ) )
+ # Check devices
+ devices = controller.CLI.devices()
+ devices = json.loads( devices )
+ availableDeviceNum = 0
+ for device in devices:
+ if device[ 'available' ] == True:
+ availableDeviceNum += 1
+ if not availableDeviceNum == upDeviceNum:
+ checkResult = EventStates().FAIL
+ main.log.warn( "Topo Check - device number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upDeviceNum, availableDeviceNum ) )
+ # Check hosts
+ hosts = controller.CLI.hosts()
+ hosts = json.loads( hosts )
+ if not len( hosts ) == upHostNum:
+ checkResult = EventStates().FAIL
+ main.log.warn( "Topo Check - host number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upHostNum, len( hosts ) ) )
+ # Check clusters
+ clusters = controller.CLI.clusters()
+ clusters = json.loads( clusters )
+ if not len( clusters ) == clusterNum:
+ checkResult = EventStates().FAIL
+ main.log.warn( "Topo Check - cluster number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, clusterNum, len( clusters ) ) )
+ return checkResult
+
+class ONOSCheck( CheckEvent ):
+ def __init__( self ):
+ CheckEvent.__init__( self )
+ self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
+ self.typeIndex= int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
+
+ def startCheckEvent( self, args=None ):
+ import json
+ checkResult = EventStates().PASS
+ topics = []
+ # TODO: Other topics?
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ dpidToAvailability = {}
+ dpidToMaster = {}
+ for device in main.devices:
+ if device.isDown() or device.isRemoved():
+ dpidToAvailability[ device.dpid ] = False
+ else:
+ dpidToAvailability[ device.dpid ] = True
+ dpidToMaster[ device.dpid ] = 'unknown'
+ # Check mastership, leaders and node states on each controller node
+ for controller in main.controllers:
+ if controller.isUp():
+ # Check mastership
+ with controller.CLILock:
+ roles = controller.CLI.roles()
+ roles = json.loads( roles )
+ for device in roles:
+ dpid = device[ 'id' ]
+ if dpidToMaster[ dpid ] == 'unknown':
+ dpidToMaster[ dpid ] = device[ 'master' ]
+ elif dpidToMaster[ dpid ] != device[ 'master' ]:
+ checkResult = EventStates().FAIL
+ main.log.warn( "ONOS Check - Mastership of %s on ONOS%s is inconsistent with that on ONOS1" % ( device.name, controller.index ) )
+ if dpidToAvailability[ dpid ] and device[ 'master' ] == "none":
+ checkResult = EventStates().FAIL
+ main.log.warn( "ONOS Check - Device %s has no master on ONOS%s" % ( device.name, controller.index ) )
+ # Check leaders
+ with controller.CLILock:
+ leaders = controller.CLI.leaders()
+ leaders = json.loads( leaders )
+ ONOSTopics = [ j['topic'] for j in leaders ]
+ for topic in topics:
+ if topic not in ONOSTopics:
+ checkResult = EventStates().FAIL
+ main.log.warn( "ONOS Check - Topic %s not in leaders on ONOS%s" % ( topic, controller.index ) )
+ # Check node state
+ with controller.CLILock:
+ nodes = controller.CLI.nodes()
+ nodes = json.loads( nodes )
+ ipToState = {}
+ for node in nodes:
+ ipToState[ node[ 'ip' ] ] = node[ 'state' ]
+ for c in main.controllers:
+ if c.isUp() and ipToState[ c.ip ] == 'READY':
+ pass
+ elif not c.isUp() and ipToState[ c.ip ] == 'INACTIVE':
+ pass
+ else:
+ checkResult = EventStates().FAIL
+ main.log.warn( "ONOS Check - ONOS%s shows wrong node state: ONOS%s is %s but state is %s" % ( controller.index, c.index, c.status, ipToState[ c.ip ] ) )
+ # TODO: check partitions?
+ return checkResult
+
+class TrafficCheck( CheckEvent ):
+ def __init__( self ):
+ CheckEvent.__init__( self )
+ self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
+ self.typeIndex= int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
+
+ def startCheckEvent( self, args=None ):
+ checkResult = EventStates().PASS
+ pool = []
+ wait = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingWait' ] )
+ timeout = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingTimeout' ] )
+ dstIPv4List = {}
+ dstIPv6List = {}
+ upHosts = []
+ for host in main.hosts:
+ if host.isUp():
+ upHosts.append( host )
+ for host in upHosts:
+ dstIPv4List[ host.index ] = []
+ dstIPv6List[ host.index ] = []
+ for correspondent in host.correspondents:
+ if not correspondent in upHosts:
+ continue
+ for ipAddress in correspondent.ipAddresses:
+ if ipAddress.startswith( str( main.params[ 'TEST' ][ 'ipv6Prefix' ] ) ):
+ dstIPv6List[ host.index ].append( ipAddress )
+ elif ipAddress.startswith( str( main.params[ 'TEST' ][ 'ipv4Prefix' ] ) ):
+ dstIPv4List[ host.index ].append( ipAddress )
+ thread = main.Thread( target=host.handle.pingHostSetAlternative,
+ threadID=main.threadID,
+ name="pingHostSetAlternative",
+ args=[ dstIPv4List[ host.index ], 1 ] )
+ pool.append( thread )
+ thread.start()
+ with main.variableLock:
+ main.threadID += 1
+ for thread in pool:
+ thread.join( 10 )
+ if not thread.result:
+ checkResult = EventStates().FAIL
+ main.log.warn( "Traffic Check - ping failed" )
+
+ if not main.enableIPv6:
+ return checkResult
+ # Check ipv6 ping
+ for host in upHosts:
+ thread = main.Thread( target=host.handle.pingHostSetAlternative,
+ threadID=main.threadID,
+ name="pingHostSetAlternative",
+ args=[ dstIPv6List[ host.index ], 1, True ] )
+ pool.append( thread )
+ thread.start()
+ with main.variableLock:
+ main.threadID += 1
+ for thread in pool:
+ thread.join( 10 )
+ if not thread.result:
+ checkResult = EventStates().FAIL
+ main.log.warn( "Traffic Check - ping6 failed" )
+ return checkResult
+