blob: ccb5831605f84ee4ab4a7c407d3bdf16bd68f14e [file] [log] [blame]
You Wangdb927a52016-02-26 11:03:28 -08001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2016 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070021"""
You Wangdb927a52016-02-26 11:03:28 -080022This file contains classes for CHOTestMonkey that are related to check event
23Author: you@onlab.us
24"""
25from tests.CHOTestMonkey.dependencies.events.Event import EventType, EventStates, Event
26
Jon Hall2bb3e212017-05-24 17:07:25 -070027
You Wangdb927a52016-02-26 11:03:28 -080028class CheckEvent( Event ):
Jon Hall2bb3e212017-05-24 17:07:25 -070029
You Wangdb927a52016-02-26 11:03:28 -080030 def __init__( self ):
31 Event.__init__( self )
32
33 def startCheckEvent( self ):
34 return EventStates().PASS
35
36 def startEvent( self, args ):
37 with self.eventLock:
You Wang52163202016-07-14 16:37:15 -070038 main.log.info( "Event recorded: {} {}".format( self.typeIndex, self.typeString ) )
You Wangdb927a52016-02-26 11:03:28 -080039 result = self.startCheckEvent()
40 return result
41
Jon Hall2bb3e212017-05-24 17:07:25 -070042
You Wangdb927a52016-02-26 11:03:28 -080043class IntentCheck( CheckEvent ):
Jon Hall2bb3e212017-05-24 17:07:25 -070044
You Wangdb927a52016-02-26 11:03:28 -080045 def __init__( self ):
46 CheckEvent.__init__( self )
47 self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
48 self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
49
50 def startCheckEvent( self, args=None ):
51 checkResult = EventStates().PASS
You Wang58aa11e2016-05-17 10:35:44 -070052 intentDict = {}
You Wangdb927a52016-02-26 11:03:28 -080053 for intent in main.intents:
You Wang58aa11e2016-05-17 10:35:44 -070054 intentDict[ intent.id ] = intent.expectedState
You Wangdb927a52016-02-26 11:03:28 -080055 for controller in main.controllers:
56 if controller.isUp():
57 with controller.CLILock:
You Wang58aa11e2016-05-17 10:35:44 -070058 intentState = controller.CLI.compareIntent( intentDict )
You Wangdb927a52016-02-26 11:03:28 -080059 if not intentState:
You Wang58aa11e2016-05-17 10:35:44 -070060 main.log.warn( "Intent Check - not all intent ids and states match that on ONOS%s" % ( controller.index ) )
You Wang817990a2019-01-22 15:07:25 -080061 # FIXME: ONOS leaves intents as WITHDRAWN state occasionally and we don't consider that as a FAIL for now
62 # checkResult = EventStates().FAIL
You Wang58aa11e2016-05-17 10:35:44 -070063 return checkResult
64
Jon Hall2bb3e212017-05-24 17:07:25 -070065
You Wang58aa11e2016-05-17 10:35:44 -070066class FlowCheck( CheckEvent ):
Jon Hall2bb3e212017-05-24 17:07:25 -070067
You Wang58aa11e2016-05-17 10:35:44 -070068 def __init__( self ):
69 CheckEvent.__init__( self )
70 self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
71 self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
72
73 def startCheckEvent( self, args=None ):
74 import json
75 checkResult = EventStates().PASS
You Wang3c276252016-09-21 15:21:36 -070076 if main.enableIPv6:
You Wang62e1cf62016-09-22 17:13:03 -070077 coreFlowNum = int( main.params[ 'EVENT' ][ 'FlowCheck' ][ 'coreFlowNum6' ] )
You Wang3c276252016-09-21 15:21:36 -070078 else:
You Wang62e1cf62016-09-22 17:13:03 -070079 coreFlowNum = int( main.params[ 'EVENT' ][ 'FlowCheck' ][ 'coreFlowNum' ] )
You Wang58aa11e2016-05-17 10:35:44 -070080 for controller in main.controllers:
81 if controller.isUp():
82 with controller.CLILock:
You Wang3c276252016-09-21 15:21:36 -070083 # Check core flow number
84 for device in main.devices:
85 if device.isRemoved():
86 continue
87 coreFlowNumOnos = controller.CLI.flowAddedCount( device.dpid, core=True )
Jon Hall2bb3e212017-05-24 17:07:25 -070088 if coreFlowNumOnos is None:
You Wang3c276252016-09-21 15:21:36 -070089 main.log.warn( "Flow Check - error when trying to get flow number of %s on ONOS%s" % ( device.dpid, controller.index ) )
90 checkResult = EventStates().FAIL
91 else:
92 coreFlowNumOnos = int( coreFlowNumOnos )
93 if coreFlowNumOnos != coreFlowNum:
94 main.log.warn( "Flow Check - core flow number of %s on ONOS%s is %s" % ( device.dpid, controller.index, coreFlowNumOnos ) )
95 checkResult = EventStates().FAIL
96 # Get flows for comparison
You Wang58aa11e2016-05-17 10:35:44 -070097 flows = controller.CLI.flows()
98 try:
99 flows = json.loads( flows )
100 except ( TypeError, ValueError ):
101 main.log.exception( "Flow Check - Object not as expected: {!r}".format( flows ) )
102 return EventStates().FAIL
103 # Compare flow IDs in ONOS and Mininet
104 flowIDList = []
105 for item in flows:
106 for flow in item[ "flows" ]:
107 flowIDList.append( hex( int( flow[ 'id' ] ) ) )
108 main.log.info( "Flow Check - current flow number on ONOS%s: %s" % ( controller.index, len( flowIDList ) ) )
109 switchList = []
110 for device in main.devices:
111 switchList.append( device.name )
112 with main.mininetLock:
113 flowCompareResult = main.Mininet1.checkFlowId( switchList, flowIDList, debug=False )
114 if not flowCompareResult:
115 main.log.warn( "Flow Check - flows on ONOS%s do not match that in Mininet" % ( controller.index ) )
116 checkResult = EventStates().FAIL
117 # Check flow state
118 flowState = controller.CLI.checkFlowsState( isPENDING=False )
119 if not flowState:
120 main.log.warn( "Flow Check - not all flows are in ADDED state on ONOS%s" % ( controller.index ) )
121 checkResult = EventStates().FAIL
You Wangdb927a52016-02-26 11:03:28 -0800122 return checkResult
123
Jon Hall2bb3e212017-05-24 17:07:25 -0700124
You Wangdb927a52016-02-26 11:03:28 -0800125class TopoCheck( CheckEvent ):
Jon Hall2bb3e212017-05-24 17:07:25 -0700126
You Wangdb927a52016-02-26 11:03:28 -0800127 def __init__( self ):
128 CheckEvent.__init__( self )
129 self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
130 self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
131
132 def startCheckEvent( self, args=None ):
133 import json
134 checkResult = EventStates().PASS
135 upLinkNum = 0
136 upDeviceNum = 0
137 upHostNum = 0
138 with main.variableLock:
139 for link in main.links:
140 if not link.isDown() and not link.isRemoved():
141 upLinkNum += 1
142 for device in main.devices:
143 if not device.isDown() and not device.isRemoved():
144 upDeviceNum += 1
145 for host in main.hosts:
146 if not host.isDown() and not host.isRemoved():
147 upHostNum += 1
148 clusterNum = 1
You Wang221db322016-06-03 15:45:52 -0700149 with main.mininetLock:
You Wang9fc5ce42019-01-23 15:10:08 -0800150 graphDictMininet = main.Mininet1.getGraphDict( useId=True, switchClasses=r"(OVSSwitch)",
151 excludeNodes=[ 'bgp', 'cs', 'nat', 'dhcp', 'r' ] )
You Wangdb927a52016-02-26 11:03:28 -0800152 for controller in main.controllers:
153 if controller.isUp():
154 with controller.CLILock:
You Wang9fc5ce42019-01-23 15:10:08 -0800155 '''
Flavio Castro82ee2f62016-06-07 15:04:12 -0700156 topoState = controller.CLI.checkStatus( upDeviceNum, upLinkNum )
You Wang9fc5ce42019-01-23 15:10:08 -0800157 if not topoState:
158 main.log.warn( "Topo Check - link or device number discoverd by ONOS%s is incorrect" % ( controller.index ) )
159 checkResult = EventStates().FAIL
You Wang221db322016-06-03 15:45:52 -0700160 # Compare ONOS and Mininet topologies
161 graphDictONOS = controller.CLI.getGraphDict()
162 compareResult = main.graph.compareGraphs( graphDictONOS, graphDictMininet )
163 if not compareResult:
164 checkResult = EventStates().FAIL
165 main.log.warn( "Topo Check - ONOS and Mininet topologies do not match" )
You Wang9fc5ce42019-01-23 15:10:08 -0800166 '''
You Wang58aa11e2016-05-17 10:35:44 -0700167 try:
You Wang221db322016-06-03 15:45:52 -0700168 # Check links
You Wang58aa11e2016-05-17 10:35:44 -0700169 links = controller.CLI.links()
170 links = json.loads( links )
171 if not len( links ) == upLinkNum:
172 checkResult = EventStates().FAIL
173 main.log.warn( "Topo Check - link number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upLinkNum, len( links ) ) )
174 # Check devices
175 devices = controller.CLI.devices()
176 devices = json.loads( devices )
177 availableDeviceNum = 0
178 for device in devices:
Jon Hall2bb3e212017-05-24 17:07:25 -0700179 if device[ 'available' ]:
You Wang58aa11e2016-05-17 10:35:44 -0700180 availableDeviceNum += 1
181 if not availableDeviceNum == upDeviceNum:
182 checkResult = EventStates().FAIL
183 main.log.warn( "Topo Check - device number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upDeviceNum, availableDeviceNum ) )
184 # Check hosts
185 hosts = controller.CLI.hosts()
186 hosts = json.loads( hosts )
You Wang9fc5ce42019-01-23 15:10:08 -0800187 if hasattr( main, "expectedHosts" ):
188 hosts = [ host for host in hosts if host[ 'id' ] in main.expectedHosts[ 'onos' ].keys() ]
You Wang58aa11e2016-05-17 10:35:44 -0700189 if not len( hosts ) == upHostNum:
You Wang9fc5ce42019-01-23 15:10:08 -0800190 # checkResult = EventStates().FAIL
You Wang58aa11e2016-05-17 10:35:44 -0700191 main.log.warn( "Topo Check - host number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, upHostNum, len( hosts ) ) )
192 # Check clusters
193 clusters = controller.CLI.clusters()
194 clusters = json.loads( clusters )
195 if not len( clusters ) == clusterNum:
196 checkResult = EventStates().FAIL
197 main.log.warn( "Topo Check - cluster number discoverd by ONOS%s is incorrect: %s expected and %s actual" % ( controller.index, clusterNum, len( clusters ) ) )
198 except ( TypeError, ValueError ):
199 main.log.exception( "Flow Check - Object not as expected" )
200 return EventStates().FAIL
You Wangdb927a52016-02-26 11:03:28 -0800201 return checkResult
202
Jon Hall2bb3e212017-05-24 17:07:25 -0700203
You Wangdb927a52016-02-26 11:03:28 -0800204class ONOSCheck( CheckEvent ):
Jon Hall2bb3e212017-05-24 17:07:25 -0700205
You Wangdb927a52016-02-26 11:03:28 -0800206 def __init__( self ):
207 CheckEvent.__init__( self )
208 self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
Jon Hall2bb3e212017-05-24 17:07:25 -0700209 self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
You Wangdb927a52016-02-26 11:03:28 -0800210
211 def startCheckEvent( self, args=None ):
212 import json
213 checkResult = EventStates().PASS
214 topics = []
215 # TODO: Other topics?
216 for i in range( 14 ):
Jon Hall8dafdcc2016-09-16 10:21:25 -0700217 topics.append( "work-partition-" + str( i ) )
You Wangdb927a52016-02-26 11:03:28 -0800218 dpidToAvailability = {}
219 dpidToMaster = {}
220 for device in main.devices:
221 if device.isDown() or device.isRemoved():
222 dpidToAvailability[ device.dpid ] = False
223 else:
224 dpidToAvailability[ device.dpid ] = True
225 dpidToMaster[ device.dpid ] = 'unknown'
226 # Check mastership, leaders and node states on each controller node
227 for controller in main.controllers:
228 if controller.isUp():
229 # Check mastership
You Wang58aa11e2016-05-17 10:35:44 -0700230 try:
231 with controller.CLILock:
232 roles = controller.CLI.roles()
233 roles = json.loads( roles )
234 for device in roles:
235 dpid = device[ 'id' ]
236 if dpidToMaster[ dpid ] == 'unknown':
237 dpidToMaster[ dpid ] = device[ 'master' ]
238 elif dpidToMaster[ dpid ] != device[ 'master' ]:
239 checkResult = EventStates().FAIL
240 main.log.warn( "ONOS Check - Mastership of %s on ONOS%s is inconsistent with that on ONOS1" % ( dpid, controller.index ) )
241 if dpidToAvailability[ dpid ] and device[ 'master' ] == "none":
242 checkResult = EventStates().FAIL
243 main.log.warn( "ONOS Check - Device %s has no master on ONOS%s" % ( dpid, controller.index ) )
244 # Check leaders
245 with controller.CLILock:
246 leaders = controller.CLI.leaders()
247 leaders = json.loads( leaders )
Jon Hall2bb3e212017-05-24 17:07:25 -0700248 ONOSTopics = [ j[ 'topic' ] for j in leaders ]
You Wang58aa11e2016-05-17 10:35:44 -0700249 for topic in topics:
250 if topic not in ONOSTopics:
251 checkResult = EventStates().FAIL
252 main.log.warn( "ONOS Check - Topic %s not in leaders on ONOS%s" % ( topic, controller.index ) )
253 # Check node state
254 with controller.CLILock:
255 nodes = controller.CLI.nodes()
256 nodes = json.loads( nodes )
257 ipToState = {}
258 for node in nodes:
259 ipToState[ node[ 'ip' ] ] = node[ 'state' ]
260 for c in main.controllers:
261 if c.isUp() and ipToState[ c.ip ] == 'READY':
262 pass
You Wang817990a2019-01-22 15:07:25 -0800263 elif not c.isUp() and c.ip not in ipToState.keys():
You Wang58aa11e2016-05-17 10:35:44 -0700264 pass
265 else:
266 checkResult = EventStates().FAIL
267 main.log.warn( "ONOS Check - ONOS%s shows wrong node state: ONOS%s is %s but state is %s" % ( controller.index, c.index, c.status, ipToState[ c.ip ] ) )
268 # TODO: check partitions?
269 except ( TypeError, ValueError ):
270 main.log.exception( "ONOS Check - Object not as expected" )
271 return EventStates().FAIL
You Wang817990a2019-01-22 15:07:25 -0800272 except Exception as e:
273 main.log.exception( "ONOS Check - Uncaught Exception: {}".format( e ) )
274 return EventStates().FAIL
You Wangdb927a52016-02-26 11:03:28 -0800275 return checkResult
276
Jon Hall2bb3e212017-05-24 17:07:25 -0700277
You Wangdb927a52016-02-26 11:03:28 -0800278class TrafficCheck( CheckEvent ):
Jon Hall2bb3e212017-05-24 17:07:25 -0700279
You Wangdb927a52016-02-26 11:03:28 -0800280 def __init__( self ):
281 CheckEvent.__init__( self )
282 self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
Jon Hall2bb3e212017-05-24 17:07:25 -0700283 self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
You Wangdb927a52016-02-26 11:03:28 -0800284
285 def startCheckEvent( self, args=None ):
286 checkResult = EventStates().PASS
287 pool = []
288 wait = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingWait' ] )
289 timeout = int( main.params[ 'EVENT' ][ 'TrafficCheck' ][ 'pingTimeout' ] )
290 dstIPv4List = {}
291 dstIPv6List = {}
292 upHosts = []
293 for host in main.hosts:
294 if host.isUp():
295 upHosts.append( host )
You Wang9fc5ce42019-01-23 15:10:08 -0800296 import re
You Wangdb927a52016-02-26 11:03:28 -0800297 for host in upHosts:
298 dstIPv4List[ host.index ] = []
299 dstIPv6List[ host.index ] = []
300 for correspondent in host.correspondents:
Jon Hall2bb3e212017-05-24 17:07:25 -0700301 if correspondent not in upHosts:
You Wangdb927a52016-02-26 11:03:28 -0800302 continue
303 for ipAddress in correspondent.ipAddresses:
You Wang9fc5ce42019-01-23 15:10:08 -0800304 if re.match( str( main.params[ 'TEST' ][ 'ipv6Regex' ] ), ipAddress ):
You Wangdb927a52016-02-26 11:03:28 -0800305 dstIPv6List[ host.index ].append( ipAddress )
You Wang9fc5ce42019-01-23 15:10:08 -0800306 elif re.match( str( main.params[ 'TEST' ][ 'ipv4Regex' ] ), ipAddress ):
You Wangdb927a52016-02-26 11:03:28 -0800307 dstIPv4List[ host.index ].append( ipAddress )
You Wang9fc5ce42019-01-23 15:10:08 -0800308 if dstIPv4List[ host.index ]:
309 main.log.debug( "Check ping from host {} to {}".format( host.name, dstIPv4List[ host.index ] ) )
You Wangdb927a52016-02-26 11:03:28 -0800310 thread = main.Thread( target=host.handle.pingHostSetAlternative,
311 threadID=main.threadID,
312 name="pingHostSetAlternative",
313 args=[ dstIPv4List[ host.index ], 1 ] )
314 pool.append( thread )
315 thread.start()
316 with main.variableLock:
317 main.threadID += 1
318 for thread in pool:
319 thread.join( 10 )
320 if not thread.result:
321 checkResult = EventStates().FAIL
322 main.log.warn( "Traffic Check - ping failed" )
323
324 if not main.enableIPv6:
325 return checkResult
326 # Check ipv6 ping
327 for host in upHosts:
You Wang9fc5ce42019-01-23 15:10:08 -0800328 if dstIPv6List[ host.index ]:
329 main.log.debug( "Check ping from host {} to {}".format( host.name, dstIPv6List[ host.index ] ) )
You Wangdb927a52016-02-26 11:03:28 -0800330 thread = main.Thread( target=host.handle.pingHostSetAlternative,
331 threadID=main.threadID,
332 name="pingHostSetAlternative",
333 args=[ dstIPv6List[ host.index ], 1, True ] )
334 pool.append( thread )
335 thread.start()
336 with main.variableLock:
337 main.threadID += 1
338 for thread in pool:
339 thread.join( 10 )
340 if not thread.result:
341 checkResult = EventStates().FAIL
342 main.log.warn( "Traffic Check - ping6 failed" )
343 return checkResult
Devin Limf0822182017-09-12 14:52:57 -0700344
345class RaftLogSizeCheck( CheckEvent ):
346
347 def __init__( self ):
348 CheckEvent.__init__( self )
349 self.typeString = main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeString' ]
350 self.typeIndex = int( main.params[ 'EVENT' ][ self.__class__.__name__ ][ 'typeIndex' ] )
351
352 def startCheckEvent( self, args=None ):
353 checkResult = EventStates().PASS
354 main.log.info( "Starting checking Raft Log size" )
355 if not main.Cluster.checkPartitionSize():
356 checkResult = EventStates().FAIL
357 main.log.warn( "Raft Log Size Check - Raft log grew too big" )
358
359 return checkResult