blob: 1d4db867ffc90f8ae9287f1a10dadf97b2c1a253 [file] [log] [blame]
Jon Hall85794ff2015-07-08 14:12:30 -07001"""
2Description: This test is to determine if a single
3 instance ONOS 'cluster' can handle a restart
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAsingleInstanceRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080051 import time
Jon Halla440e872016-03-31 15:15:50 -070052 import json
Jon Hall85794ff2015-07-08 14:12:30 -070053 main.log.info( "ONOS Single node cluster restart " +
54 "HA test - initialization" )
55 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070056 main.caseExplanation = "Setup the test environment including " +\
Jon Hall85794ff2015-07-08 14:12:30 -070057 "installing ONOS, starting Mininet and ONOS" +\
58 "cli sessions."
Jon Hall85794ff2015-07-08 14:12:30 -070059
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall85794ff2015-07-08 14:12:30 -070071
Jon Halle1a3b752015-07-22 13:02:46 -070072 try:
Jon Hall53c5e662016-04-13 16:06:56 -070073 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070074 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070075 except Exception as e:
76 main.log.exception( e )
77 main.cleanup()
78 main.exit()
79
80 main.CLIs = []
81 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -070082 ipList = []
83 for i in range( 1, int( main.ONOSbench.maxNodes ) + 1 ):
84 try:
Jon Halle1a3b752015-07-22 13:02:46 -070085 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
86 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
87 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -070088 except AttributeError:
89 break
Jon Hall85794ff2015-07-08 14:12:30 -070090
Jon Hall5cf14d52015-07-16 12:15:19 -070091 main.step( "Create cell file" )
92 cellAppString = main.params[ 'ENV' ][ 'appString' ]
93 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
94 main.Mininet1.ip_address,
95 cellAppString, ipList )
Jon Hall85794ff2015-07-08 14:12:30 -070096 main.step( "Applying cell variable to environment" )
97 cellResult = main.ONOSbench.setCell( cellName )
98 verifyResult = main.ONOSbench.verifyCell()
99
100 # FIXME:this is short term fix
101 main.log.info( "Removing raft logs" )
102 main.ONOSbench.onosRemoveRaftLogs()
103
104 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700105 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700106 main.ONOSbench.onosUninstall( node.ip_address )
107
108 # Make sure ONOS is DEAD
109 main.log.info( "Killing any ONOS processes" )
110 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700111 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700112 killed = main.ONOSbench.onosKill( node.ip_address )
113 killResults = killResults and killed
114
115 cleanInstallResult = main.TRUE
116 gitPullResult = main.TRUE
117
118 main.step( "Starting Mininet" )
119 # scp topo file to mininet
120 # TODO: move to params?
121 topoName = "obelisk.py"
122 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700123 main.ONOSbench.scp( main.Mininet1,
124 filePath + topoName,
125 main.Mininet1.home,
126 direction="to" )
Jon Hall85794ff2015-07-08 14:12:30 -0700127 mnResult = main.Mininet1.startNet( )
128 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
129 onpass="Mininet Started",
130 onfail="Error starting Mininet" )
131
132 main.step( "Git checkout and pull " + gitBranch )
133 if PULLCODE:
134 main.ONOSbench.gitCheckout( gitBranch )
135 gitPullResult = main.ONOSbench.gitPull()
136 # values of 1 or 3 are good
137 utilities.assert_lesser( expect=0, actual=gitPullResult,
138 onpass="Git pull successful",
139 onfail="Git pull failed" )
140 main.ONOSbench.getVersion( report=True )
141
142 main.step( "Using mvn clean install" )
143 cleanInstallResult = main.TRUE
144 if PULLCODE and gitPullResult == main.TRUE:
145 cleanInstallResult = main.ONOSbench.cleanInstall()
146 else:
147 main.log.warn( "Did not pull new code so skipping mvn " +
148 "clean install" )
149 utilities.assert_equals( expect=main.TRUE,
150 actual=cleanInstallResult,
151 onpass="MCI successful",
152 onfail="MCI failed" )
153 # GRAPHS
154 # NOTE: important params here:
155 # job = name of Jenkins job
156 # Plot Name = Plot-HA, only can be used if multiple plots
157 # index = The number of the graph under plot name
Jon Hall5cf14d52015-07-16 12:15:19 -0700158 job = "HAsingleInstanceRestart"
Jon Hall85794ff2015-07-08 14:12:30 -0700159 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700160 index = "2"
Jon Hall85794ff2015-07-08 14:12:30 -0700161 graphs = '<ac:structured-macro ac:name="html">\n'
162 graphs += '<ac:plain-text-body><![CDATA[\n'
163 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800164 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall85794ff2015-07-08 14:12:30 -0700165 '&width=500&height=300"' +\
166 'noborder="0" width="500" height="300" scrolling="yes" ' +\
167 'seamless="seamless"></iframe>\n'
168 graphs += ']]></ac:plain-text-body>\n'
169 graphs += '</ac:structured-macro>\n'
170 main.log.wiki(graphs)
171
Jon Halle1a3b752015-07-22 13:02:46 -0700172 main.CLIs = []
173 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700174 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700175 for i in range( 1, main.numCtrls + 1 ):
176 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
177 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
178 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700179
180 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, "SingleHA",
181 main.Mininet1.ip_address,
182 cellAppString, ipList[ 0 ] )
Jon Hall85794ff2015-07-08 14:12:30 -0700183 cellResult = main.ONOSbench.setCell( "SingleHA" )
184 verifyResult = main.ONOSbench.verifyCell()
185 main.step( "Creating ONOS package" )
186 packageResult = main.ONOSbench.onosPackage()
187 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
188 onpass="ONOS package successful",
189 onfail="ONOS package failed" )
190
191 main.step( "Installing ONOS package" )
Jon Halla440e872016-03-31 15:15:50 -0700192 onosInstallResult = main.TRUE
193 for node in main.nodes:
194 tmpResult = main.ONOSbench.onosInstall( options="-f",
195 node=node.ip_address )
196 onosInstallResult = onosInstallResult and tmpResult
Jon Hall85794ff2015-07-08 14:12:30 -0700197 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
198 onpass="ONOS install successful",
199 onfail="ONOS install failed" )
200
201 main.step( "Checking if ONOS is up yet" )
202 for i in range( 2 ):
Jon Halla440e872016-03-31 15:15:50 -0700203 onosIsupResult = main.TRUE
204 for node in main.nodes:
205 started = main.ONOSbench.isup( node.ip_address )
206 if not started:
207 main.log.error( node.name + " hasn't started" )
208 onosIsupResult = onosIsupResult and started
209 if onosIsupResult == main.TRUE:
Jon Hall85794ff2015-07-08 14:12:30 -0700210 break
Jon Halla440e872016-03-31 15:15:50 -0700211 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
Jon Hall85794ff2015-07-08 14:12:30 -0700212 onpass="ONOS startup successful",
213 onfail="ONOS startup failed" )
214
215 main.log.step( "Starting ONOS CLI sessions" )
Jon Halla440e872016-03-31 15:15:50 -0700216 cliResults = main.TRUE
217 threads = []
218 for i in range( main.numCtrls ):
219 t = main.Thread( target=main.CLIs[i].startOnosCli,
220 name="startOnosCli-" + str( i ),
221 args=[main.nodes[i].ip_address] )
222 threads.append( t )
223 t.start()
224
225 for t in threads:
226 t.join()
227 cliResults = cliResults and t.result
Jon Hall85794ff2015-07-08 14:12:30 -0700228 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
229 onpass="ONOS cli startup successful",
230 onfail="ONOS cli startup failed" )
231
Jon Halla440e872016-03-31 15:15:50 -0700232 # Create a list of active nodes for use when some nodes are stopped
233 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
234
Jon Hall85794ff2015-07-08 14:12:30 -0700235 if main.params[ 'tcpdump' ].lower() == "true":
236 main.step( "Start Packet Capture MN" )
237 main.Mininet2.startTcpdump(
238 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
239 + "-MN.pcap",
240 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
241 port=main.params[ 'MNtcpdump' ][ 'port' ] )
242
Jon Halla440e872016-03-31 15:15:50 -0700243 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700244 nodeResults = utilities.retry( main.HA.nodesCheck,
245 False,
246 args=[main.activeNodes],
247 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700248
Jon Hall41d39f12016-04-11 22:54:35 -0700249 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700250 onpass="Nodes check successful",
251 onfail="Nodes check NOT successful" )
252
253 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700254 for i in main.activeNodes:
255 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700256 main.log.debug( "{} components not ACTIVE: \n{}".format(
257 cli.name,
258 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700259 main.log.error( "Failed to start ONOS, stopping test" )
260 main.cleanup()
261 main.exit()
262
Jon Hall172b7ba2016-04-07 18:12:20 -0700263 main.step( "Activate apps defined in the params file" )
264 # get data from the params
265 apps = main.params.get( 'apps' )
266 if apps:
267 apps = apps.split(',')
268 main.log.warn( apps )
269 activateResult = True
270 for app in apps:
271 main.CLIs[ 0 ].app( app, "Activate" )
272 # TODO: check this worked
273 time.sleep( 10 ) # wait for apps to activate
274 for app in apps:
275 state = main.CLIs[ 0 ].appStatus( app )
276 if state == "ACTIVE":
277 activateResult = activeResult and True
278 else:
279 main.log.error( "{} is in {} state".format( app, state ) )
280 activeResult = False
281 utilities.assert_equals( expect=True,
282 actual=activateResult,
283 onpass="Successfully activated apps",
284 onfail="Failed to activate apps" )
285 else:
286 main.log.warn( "No apps were specified to be loaded after startup" )
287
288 main.step( "Set ONOS configurations" )
289 config = main.params.get( 'ONOS_Configuration' )
290 if config:
291 main.log.debug( config )
292 checkResult = main.TRUE
293 for component in config:
294 for setting in config[component]:
295 value = config[component][setting]
296 check = main.CLIs[ 0 ].setCfg( component, setting, value )
297 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
298 checkResult = check and checkResult
299 utilities.assert_equals( expect=main.TRUE,
300 actual=checkResult,
301 onpass="Successfully set config",
302 onfail="Failed to set config" )
303 else:
304 main.log.warn( "No configurations were specified to be changed after startup" )
305
Jon Hall9d2dcad2016-04-08 10:15:20 -0700306 main.step( "App Ids check" )
307 appCheck = main.TRUE
308 threads = []
309 for i in main.activeNodes:
310 t = main.Thread( target=main.CLIs[i].appToIDCheck,
311 name="appToIDCheck-" + str( i ),
312 args=[] )
313 threads.append( t )
314 t.start()
315
316 for t in threads:
317 t.join()
318 appCheck = appCheck and t.result
319 if appCheck != main.TRUE:
320 node = main.activeNodes[0]
321 main.log.warn( main.CLIs[node].apps() )
322 main.log.warn( main.CLIs[node].appIDs() )
323 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
324 onpass="App Ids seem to be correct",
325 onfail="Something is wrong with app Ids" )
326
Jon Hall85794ff2015-07-08 14:12:30 -0700327 def CASE2( self, main ):
328 """
329 Assign devices to controllers
330 """
331 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700332 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700333 assert main, "main not defined"
334 assert utilities.assert_equals, "utilities.assert_equals not defined"
335
336 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700337 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700338 "and check that an ONOS node becomes the " +\
339 "master of the device."
340 main.step( "Assign switches to controllers" )
341
342 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700343 for i in range( main.numCtrls ):
344 ipList.append( main.nodes[ i ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700345 swList = []
346 for i in range( 1, 29 ):
347 swList.append( "s" + str( i ) )
348 main.Mininet1.assignSwController( sw=swList, ip=ipList )
349
350 mastershipCheck = main.TRUE
351 for i in range( 1, 29 ):
352 response = main.Mininet1.getSwController( "s" + str( i ) )
353 try:
354 main.log.info( str( response ) )
355 except Exception:
356 main.log.info( repr( response ) )
Jon Halla440e872016-03-31 15:15:50 -0700357 for node in main.nodes:
358 if re.search( "tcp:" + node.ip_address, response ):
359 mastershipCheck = mastershipCheck and main.TRUE
360 else:
361 main.log.error( "Error, node " + node.ip_address + " is " +
362 "not in the list of controllers s" +
363 str( i ) + " is connecting to." )
364 mastershipCheck = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700365 utilities.assert_equals(
366 expect=main.TRUE,
367 actual=mastershipCheck,
368 onpass="Switch mastership assigned correctly",
369 onfail="Switches not assigned correctly to controllers" )
370
371 def CASE21( self, main ):
372 """
373 Assign mastership to controllers
374 """
Jon Halle1a3b752015-07-22 13:02:46 -0700375 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700376 assert main, "main not defined"
377 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700378 assert main.CLIs, "main.CLIs not defined"
379 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700380
381 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700382 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700383 "device. Then manually assign" +\
384 " mastership to specific ONOS nodes using" +\
385 " 'device-role'"
386 main.step( "Assign mastership of switches to specific controllers" )
Jon Halla440e872016-03-31 15:15:50 -0700387 # Manually assign mastership to the controller we want
Jon Hall85794ff2015-07-08 14:12:30 -0700388 roleCall = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700389
390 ipList = [ ]
391 deviceList = []
392 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall85794ff2015-07-08 14:12:30 -0700393 try:
Jon Halla440e872016-03-31 15:15:50 -0700394 # Assign mastership to specific controllers. This assignment was
395 # determined for a 7 node cluser, but will work with any sized
396 # cluster
Jon Hall85794ff2015-07-08 14:12:30 -0700397 for i in range( 1, 29 ): # switches 1 through 28
Jon Hall85794ff2015-07-08 14:12:30 -0700398 # set up correct variables:
399 if i == 1:
Jon Halla440e872016-03-31 15:15:50 -0700400 c = 0
401 ip = main.nodes[ c ].ip_address # ONOS1
402 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700403 elif i == 2:
Jon Halla440e872016-03-31 15:15:50 -0700404 c = 1 % main.numCtrls
405 ip = main.nodes[ c ].ip_address # ONOS2
406 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700407 elif i == 3:
Jon Halla440e872016-03-31 15:15:50 -0700408 c = 1 % main.numCtrls
409 ip = main.nodes[ c ].ip_address # ONOS2
410 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700411 elif i == 4:
Jon Halla440e872016-03-31 15:15:50 -0700412 c = 3 % main.numCtrls
413 ip = main.nodes[ c ].ip_address # ONOS4
414 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700415 elif i == 5:
Jon Halla440e872016-03-31 15:15:50 -0700416 c = 2 % main.numCtrls
417 ip = main.nodes[ c ].ip_address # ONOS3
418 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700419 elif i == 6:
Jon Halla440e872016-03-31 15:15:50 -0700420 c = 2 % main.numCtrls
421 ip = main.nodes[ c ].ip_address # ONOS3
422 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700423 elif i == 7:
Jon Halla440e872016-03-31 15:15:50 -0700424 c = 5 % main.numCtrls
425 ip = main.nodes[ c ].ip_address # ONOS6
426 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700427 elif i >= 8 and i <= 17:
Jon Halla440e872016-03-31 15:15:50 -0700428 c = 4 % main.numCtrls
429 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall85794ff2015-07-08 14:12:30 -0700430 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700431 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700432 elif i >= 18 and i <= 27:
Jon Halla440e872016-03-31 15:15:50 -0700433 c = 6 % main.numCtrls
434 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall85794ff2015-07-08 14:12:30 -0700435 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700436 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700437 elif i == 28:
Jon Halla440e872016-03-31 15:15:50 -0700438 c = 0
439 ip = main.nodes[ c ].ip_address # ONOS1
440 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700441 else:
442 main.log.error( "You didn't write an else statement for " +
443 "switch s" + str( i ) )
Jon Halla440e872016-03-31 15:15:50 -0700444 roleCall = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700445 # Assign switch
446 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
447 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700448 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
449 ipList.append( ip )
450 deviceList.append( deviceId )
Jon Hall85794ff2015-07-08 14:12:30 -0700451 except ( AttributeError, AssertionError ):
452 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700453 main.log.info( onosCli.devices() )
Jon Hall85794ff2015-07-08 14:12:30 -0700454 utilities.assert_equals(
455 expect=main.TRUE,
456 actual=roleCall,
457 onpass="Re-assigned switch mastership to designated controller",
458 onfail="Something wrong with deviceRole calls" )
459
460 main.step( "Check mastership was correctly assigned" )
Jon Halla440e872016-03-31 15:15:50 -0700461 roleCheck = main.TRUE
462 # NOTE: This is due to the fact that device mastership change is not
463 # atomic and is actually a multi step process
464 time.sleep( 5 )
465 for i in range( len( ipList ) ):
466 ip = ipList[i]
467 deviceId = deviceList[i]
468 # Check assignment
469 master = onosCli.getRole( deviceId ).get( 'master' )
470 if ip in master:
471 roleCheck = roleCheck and main.TRUE
472 else:
473 roleCheck = roleCheck and main.FALSE
474 main.log.error( "Error, controller " + ip + " is not" +
475 " master " + "of device " +
476 str( deviceId ) + ". Master is " +
477 repr( master ) + "." )
Jon Hall85794ff2015-07-08 14:12:30 -0700478 utilities.assert_equals(
479 expect=main.TRUE,
480 actual=roleCheck,
481 onpass="Switches were successfully reassigned to designated " +
482 "controller",
483 onfail="Switches were not successfully reassigned" )
484
485 def CASE3( self, main ):
486 """
487 Assign intents
488 """
489 import time
490 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700491 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700492 assert main, "main not defined"
493 assert utilities.assert_equals, "utilities.assert_equals not defined"
494 # NOTE: we must reinstall intents until we have a persistant intent
495 # datastore!
496 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700497 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700498 "assign predetermined host-to-host intents." +\
499 " After installation, check that the intent" +\
500 " is distributed to all nodes and the state" +\
501 " is INSTALLED"
502
503 # install onos-app-fwd
504 main.step( "Install reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700505 onosCli = main.CLIs[ main.activeNodes[0] ]
506 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700507 utilities.assert_equals( expect=main.TRUE, actual=installResults,
508 onpass="Install fwd successful",
509 onfail="Install fwd failed" )
510
511 main.step( "Check app ids" )
Jon Halla440e872016-03-31 15:15:50 -0700512 appCheck = main.TRUE
513 threads = []
514 for i in main.activeNodes:
515 t = main.Thread( target=main.CLIs[i].appToIDCheck,
516 name="appToIDCheck-" + str( i ),
517 args=[] )
518 threads.append( t )
519 t.start()
520
521 for t in threads:
522 t.join()
523 appCheck = appCheck and t.result
Jon Hall85794ff2015-07-08 14:12:30 -0700524 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700525 main.log.warn( onosCli.apps() )
526 main.log.warn( onosCli.appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700527 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
528 onpass="App Ids seem to be correct",
529 onfail="Something is wrong with app Ids" )
530
531 main.step( "Discovering Hosts( Via pingall for now )" )
532 # FIXME: Once we have a host discovery mechanism, use that instead
533 # REACTIVE FWD test
534 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700535 passMsg = "Reactive Pingall test passed"
536 time1 = time.time()
537 pingResult = main.Mininet1.pingall()
538 time2 = time.time()
539 if not pingResult:
540 main.log.warn("First pingall failed. Trying again...")
Jon Hall85794ff2015-07-08 14:12:30 -0700541 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700542 passMsg += " on the second try"
543 utilities.assert_equals(
544 expect=main.TRUE,
545 actual=pingResult,
546 onpass= passMsg,
547 onfail="Reactive Pingall failed, " +
548 "one or more ping pairs failed" )
549 main.log.info( "Time for pingall: %2f seconds" %
550 ( time2 - time1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700551 # timeout for fwd flows
552 time.sleep( 11 )
553 # uninstall onos-app-fwd
554 main.step( "Uninstall reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700555 node = main.activeNodes[0]
556 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700557 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
558 onpass="Uninstall fwd successful",
559 onfail="Uninstall fwd failed" )
560
561 main.step( "Check app ids" )
Jon Halla440e872016-03-31 15:15:50 -0700562 threads = []
563 appCheck2 = main.TRUE
564 for i in main.activeNodes:
565 t = main.Thread( target=main.CLIs[i].appToIDCheck,
566 name="appToIDCheck-" + str( i ),
567 args=[] )
568 threads.append( t )
569 t.start()
570
571 for t in threads:
572 t.join()
573 appCheck2 = appCheck2 and t.result
Jon Hall85794ff2015-07-08 14:12:30 -0700574 if appCheck2 != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700575 node = main.activeNodes[0]
576 main.log.warn( main.CLIs[node].apps() )
577 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700578 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
579 onpass="App Ids seem to be correct",
580 onfail="Something is wrong with app Ids" )
581
582 main.step( "Add host intents via cli" )
583 intentIds = []
Jon Halla440e872016-03-31 15:15:50 -0700584 # TODO: move the host numbers to params
585 # Maybe look at all the paths we ping?
Jon Hall85794ff2015-07-08 14:12:30 -0700586 intentAddResult = True
587 hostResult = main.TRUE
588 for i in range( 8, 18 ):
589 main.log.info( "Adding host intent between h" + str( i ) +
590 " and h" + str( i + 10 ) )
591 host1 = "00:00:00:00:00:" + \
592 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
593 host2 = "00:00:00:00:00:" + \
594 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
595 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700596 host1Dict = onosCli.getHost( host1 )
597 host2Dict = onosCli.getHost( host2 )
Jon Hall85794ff2015-07-08 14:12:30 -0700598 host1Id = None
599 host2Id = None
600 if host1Dict and host2Dict:
601 host1Id = host1Dict.get( 'id', None )
602 host2Id = host2Dict.get( 'id', None )
603 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700604 nodeNum = ( i % len( main.activeNodes ) )
605 node = main.activeNodes[nodeNum]
606 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall85794ff2015-07-08 14:12:30 -0700607 if tmpId:
608 main.log.info( "Added intent with id: " + tmpId )
609 intentIds.append( tmpId )
610 else:
611 main.log.error( "addHostIntent returned: " +
612 repr( tmpId ) )
613 else:
614 main.log.error( "Error, getHost() failed for h" + str( i ) +
615 " and/or h" + str( i + 10 ) )
Jon Halla440e872016-03-31 15:15:50 -0700616 node = main.activeNodes[0]
617 hosts = main.CLIs[node].hosts()
Jon Hall85794ff2015-07-08 14:12:30 -0700618 main.log.warn( "Hosts output: " )
619 try:
620 main.log.warn( json.dumps( json.loads( hosts ),
621 sort_keys=True,
622 indent=4,
623 separators=( ',', ': ' ) ) )
624 except ( ValueError, TypeError ):
625 main.log.warn( repr( hosts ) )
626 hostResult = main.FALSE
627 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
628 onpass="Found a host id for each host",
629 onfail="Error looking up host ids" )
630
631 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700632 onosIds = onosCli.getAllIntentsId()
Jon Hall85794ff2015-07-08 14:12:30 -0700633 main.log.info( "Submitted intents: " + str( intentIds ) )
634 main.log.info( "Intents in ONOS: " + str( onosIds ) )
635 for intent in intentIds:
636 if intent in onosIds:
637 pass # intent submitted is in onos
638 else:
639 intentAddResult = False
640 if intentAddResult:
641 intentStop = time.time()
642 else:
643 intentStop = None
644 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700645 intents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -0700646 intentStates = []
647 installedCheck = True
648 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
649 count = 0
650 try:
651 for intent in json.loads( intents ):
652 state = intent.get( 'state', None )
653 if "INSTALLED" not in state:
654 installedCheck = False
655 intentId = intent.get( 'id', None )
656 intentStates.append( ( intentId, state ) )
657 except ( ValueError, TypeError ):
658 main.log.exception( "Error parsing intents" )
659 # add submitted intents not in the store
660 tmplist = [ i for i, s in intentStates ]
661 missingIntents = False
662 for i in intentIds:
663 if i not in tmplist:
664 intentStates.append( ( i, " - " ) )
665 missingIntents = True
666 intentStates.sort()
667 for i, s in intentStates:
668 count += 1
669 main.log.info( "%-6s%-15s%-15s" %
670 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700671 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -0700672 try:
673 missing = False
674 if leaders:
675 parsedLeaders = json.loads( leaders )
676 main.log.warn( json.dumps( parsedLeaders,
677 sort_keys=True,
678 indent=4,
679 separators=( ',', ': ' ) ) )
680 # check for all intent partitions
681 topics = []
682 for i in range( 14 ):
683 topics.append( "intent-partition-" + str( i ) )
684 main.log.debug( topics )
685 ONOStopics = [ j['topic'] for j in parsedLeaders ]
686 for topic in topics:
687 if topic not in ONOStopics:
688 main.log.error( "Error: " + topic +
689 " not in leaders" )
690 missing = True
691 else:
692 main.log.error( "leaders() returned None" )
693 except ( ValueError, TypeError ):
694 main.log.exception( "Error parsing leaders" )
695 main.log.error( repr( leaders ) )
696 # Check all nodes
697 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700698 for i in main.activeNodes:
699 response = main.CLIs[i].leaders( jsonFormat=False)
700 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
701 str( response ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700702
Jon Halla440e872016-03-31 15:15:50 -0700703 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -0700704 try:
705 if partitions :
706 parsedPartitions = json.loads( partitions )
707 main.log.warn( json.dumps( parsedPartitions,
708 sort_keys=True,
709 indent=4,
710 separators=( ',', ': ' ) ) )
711 # TODO check for a leader in all paritions
712 # TODO check for consistency among nodes
713 else:
714 main.log.error( "partitions() returned None" )
715 except ( ValueError, TypeError ):
716 main.log.exception( "Error parsing partitions" )
717 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700718 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -0700719 try:
720 if pendingMap :
721 parsedPending = json.loads( pendingMap )
722 main.log.warn( json.dumps( parsedPending,
723 sort_keys=True,
724 indent=4,
725 separators=( ',', ': ' ) ) )
726 # TODO check something here?
727 else:
728 main.log.error( "pendingMap() returned None" )
729 except ( ValueError, TypeError ):
730 main.log.exception( "Error parsing pending map" )
731 main.log.error( repr( pendingMap ) )
732
733 intentAddResult = bool( intentAddResult and not missingIntents and
734 installedCheck )
735 if not intentAddResult:
736 main.log.error( "Error in pushing host intents to ONOS" )
737
738 main.step( "Intent Anti-Entropy dispersion" )
Jon Halla440e872016-03-31 15:15:50 -0700739 for j in range(100):
Jon Hall85794ff2015-07-08 14:12:30 -0700740 correct = True
741 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700742 for i in main.activeNodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700743 onosIds = []
Jon Halla440e872016-03-31 15:15:50 -0700744 ids = main.CLIs[i].getAllIntentsId()
Jon Hall85794ff2015-07-08 14:12:30 -0700745 onosIds.append( ids )
Jon Halla440e872016-03-31 15:15:50 -0700746 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall85794ff2015-07-08 14:12:30 -0700747 str( sorted( onosIds ) ) )
748 if sorted( ids ) != sorted( intentIds ):
749 main.log.warn( "Set of intent IDs doesn't match" )
750 correct = False
751 break
752 else:
Jon Halla440e872016-03-31 15:15:50 -0700753 intents = json.loads( main.CLIs[i].intents() )
Jon Hall85794ff2015-07-08 14:12:30 -0700754 for intent in intents:
755 if intent[ 'state' ] != "INSTALLED":
756 main.log.warn( "Intent " + intent[ 'id' ] +
757 " is " + intent[ 'state' ] )
758 correct = False
759 break
760 if correct:
761 break
762 else:
763 time.sleep(1)
764 if not intentStop:
765 intentStop = time.time()
766 global gossipTime
767 gossipTime = intentStop - intentStart
768 main.log.info( "It took about " + str( gossipTime ) +
769 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700770 gossipPeriod = int( main.params['timers']['gossip'] )
Jon Halla440e872016-03-31 15:15:50 -0700771 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall85794ff2015-07-08 14:12:30 -0700772 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700773 expect=maxGossipTime, actual=gossipTime,
Jon Hall85794ff2015-07-08 14:12:30 -0700774 onpass="ECM anti-entropy for intents worked within " +
775 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700776 onfail="Intent ECM anti-entropy took too long. " +
777 "Expected time:{}, Actual time:{}".format( maxGossipTime,
778 gossipTime ) )
779 if gossipTime <= maxGossipTime:
Jon Hall85794ff2015-07-08 14:12:30 -0700780 intentAddResult = True
781
782 if not intentAddResult or "key" in pendingMap:
783 import time
784 installedCheck = True
785 main.log.info( "Sleeping 60 seconds to see if intents are found" )
786 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700787 onosIds = onosCli.getAllIntentsId()
Jon Hall85794ff2015-07-08 14:12:30 -0700788 main.log.info( "Submitted intents: " + str( intentIds ) )
789 main.log.info( "Intents in ONOS: " + str( onosIds ) )
790 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700791 intents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -0700792 intentStates = []
793 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
794 count = 0
795 try:
796 for intent in json.loads( intents ):
797 # Iter through intents of a node
798 state = intent.get( 'state', None )
799 if "INSTALLED" not in state:
800 installedCheck = False
801 intentId = intent.get( 'id', None )
802 intentStates.append( ( intentId, state ) )
803 except ( ValueError, TypeError ):
804 main.log.exception( "Error parsing intents" )
805 # add submitted intents not in the store
806 tmplist = [ i for i, s in intentStates ]
807 for i in intentIds:
808 if i not in tmplist:
809 intentStates.append( ( i, " - " ) )
810 intentStates.sort()
811 for i, s in intentStates:
812 count += 1
813 main.log.info( "%-6s%-15s%-15s" %
814 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700815 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -0700816 try:
817 missing = False
818 if leaders:
819 parsedLeaders = json.loads( leaders )
820 main.log.warn( json.dumps( parsedLeaders,
821 sort_keys=True,
822 indent=4,
823 separators=( ',', ': ' ) ) )
824 # check for all intent partitions
825 # check for election
826 topics = []
827 for i in range( 14 ):
828 topics.append( "intent-partition-" + str( i ) )
829 # FIXME: this should only be after we start the app
830 topics.append( "org.onosproject.election" )
831 main.log.debug( topics )
832 ONOStopics = [ j['topic'] for j in parsedLeaders ]
833 for topic in topics:
834 if topic not in ONOStopics:
835 main.log.error( "Error: " + topic +
836 " not in leaders" )
837 missing = True
838 else:
839 main.log.error( "leaders() returned None" )
840 except ( ValueError, TypeError ):
841 main.log.exception( "Error parsing leaders" )
842 main.log.error( repr( leaders ) )
843 # Check all nodes
844 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700845 for i in main.activeNodes:
846 node = main.CLIs[i]
847 response = node.leaders( jsonFormat=False)
848 main.log.warn( str( node.name ) + " leaders output: \n" +
849 str( response ) )
850
851 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -0700852 try:
853 if partitions :
854 parsedPartitions = json.loads( partitions )
855 main.log.warn( json.dumps( parsedPartitions,
856 sort_keys=True,
857 indent=4,
858 separators=( ',', ': ' ) ) )
859 # TODO check for a leader in all paritions
860 # TODO check for consistency among nodes
861 else:
862 main.log.error( "partitions() returned None" )
863 except ( ValueError, TypeError ):
864 main.log.exception( "Error parsing partitions" )
865 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700866 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -0700867 try:
868 if pendingMap :
869 parsedPending = json.loads( pendingMap )
870 main.log.warn( json.dumps( parsedPending,
871 sort_keys=True,
872 indent=4,
873 separators=( ',', ': ' ) ) )
874 # TODO check something here?
875 else:
876 main.log.error( "pendingMap() returned None" )
877 except ( ValueError, TypeError ):
878 main.log.exception( "Error parsing pending map" )
879 main.log.error( repr( pendingMap ) )
880
881 def CASE4( self, main ):
882 """
883 Ping across added host intents
884 """
885 import json
886 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700887 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700888 assert main, "main not defined"
889 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halla440e872016-03-31 15:15:50 -0700890 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700891 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700892 "functionality and check the state of " +\
893 "the intent"
Jon Hall9d2dcad2016-04-08 10:15:20 -0700894
Jon Hall41d39f12016-04-11 22:54:35 -0700895 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall9d2dcad2016-04-08 10:15:20 -0700896 main.step( "Check Intent state" )
897 installedCheck = True
898 # Print the intent states
899 intents = main.ONOScli1.intents()
900 intentStates = []
901 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
902 count = 0
903 # Iter through intents of a node
904 try:
905 for intent in json.loads( intents ):
906 state = intent.get( 'state', None )
907 if "INSTALLED" not in state:
908 installedCheck = False
909 intentId = intent.get( 'id', None )
910 intentStates.append( ( intentId, state ) )
911 except ( ValueError, TypeError ):
912 main.log.exception( "Error parsing intents." )
913 # Print states
914 intentStates.sort()
915 for i, s in intentStates:
916 count += 1
917 main.log.info( "%-6s%-15s%-15s" %
918 ( str( count ), str( i ), str( s ) ) )
919 utilities.assert_equals( expect=True, actual=installedCheck,
920 onpass="Intents are all INSTALLED",
921 onfail="Intents are not all in " +
922 "INSTALLED state" )
923
Jon Hall85794ff2015-07-08 14:12:30 -0700924 main.step( "Ping across added host intents" )
925 PingResult = main.TRUE
926 for i in range( 8, 18 ):
927 ping = main.Mininet1.pingHost( src="h" + str( i ),
928 target="h" + str( i + 10 ) )
929 PingResult = PingResult and ping
930 if ping == main.FALSE:
931 main.log.warn( "Ping failed between h" + str( i ) +
932 " and h" + str( i + 10 ) )
933 elif ping == main.TRUE:
934 main.log.info( "Ping test passed!" )
935 # Don't set PingResult or you'd override failures
936 if PingResult == main.FALSE:
937 main.log.error(
938 "Intents have not been installed correctly, pings failed." )
939 # TODO: pretty print
940 main.log.warn( "ONOS1 intents: " )
941 try:
Jon Halla440e872016-03-31 15:15:50 -0700942 tmpIntents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -0700943 main.log.warn( json.dumps( json.loads( tmpIntents ),
944 sort_keys=True,
945 indent=4,
946 separators=( ',', ': ' ) ) )
947 except ( ValueError, TypeError ):
948 main.log.warn( repr( tmpIntents ) )
949 utilities.assert_equals(
950 expect=main.TRUE,
951 actual=PingResult,
952 onpass="Intents have been installed correctly and pings work",
953 onfail="Intents have not been installed correctly, pings failed." )
954
Jon Hall85794ff2015-07-08 14:12:30 -0700955 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -0700956 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -0700957 topicCheck = main.TRUE
958 try:
959 if leaders:
960 parsedLeaders = json.loads( leaders )
961 main.log.warn( json.dumps( parsedLeaders,
962 sort_keys=True,
963 indent=4,
964 separators=( ',', ': ' ) ) )
965 # check for all intent partitions
966 # check for election
967 # TODO: Look at Devices as topics now that it uses this system
968 topics = []
969 for i in range( 14 ):
970 topics.append( "intent-partition-" + str( i ) )
971 # FIXME: this should only be after we start the app
972 # FIXME: topics.append( "org.onosproject.election" )
973 # Print leaders output
974 main.log.debug( topics )
975 ONOStopics = [ j['topic'] for j in parsedLeaders ]
976 for topic in topics:
977 if topic not in ONOStopics:
978 main.log.error( "Error: " + topic +
979 " not in leaders" )
980 topicCheck = main.FALSE
981 else:
982 main.log.error( "leaders() returned None" )
983 topicCheck = main.FALSE
984 except ( ValueError, TypeError ):
985 topicCheck = main.FALSE
986 main.log.exception( "Error parsing leaders" )
987 main.log.error( repr( leaders ) )
988 # TODO: Check for a leader of these topics
Jon Halla440e872016-03-31 15:15:50 -0700989 # Check all nodes
990 if topicCheck:
991 for i in main.activeNodes:
992 node = main.CLIs[i]
993 response = node.leaders( jsonFormat=False)
994 main.log.warn( str( node.name ) + " leaders output: \n" +
995 str( response ) )
996
Jon Hall85794ff2015-07-08 14:12:30 -0700997 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
998 onpass="intent Partitions is in leaders",
999 onfail="Some topics were lost " )
1000 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -07001001 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -07001002 try:
1003 if partitions :
1004 parsedPartitions = json.loads( partitions )
1005 main.log.warn( json.dumps( parsedPartitions,
1006 sort_keys=True,
1007 indent=4,
1008 separators=( ',', ': ' ) ) )
1009 # TODO check for a leader in all paritions
1010 # TODO check for consistency among nodes
1011 else:
1012 main.log.error( "partitions() returned None" )
1013 except ( ValueError, TypeError ):
1014 main.log.exception( "Error parsing partitions" )
1015 main.log.error( repr( partitions ) )
1016 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001017 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -07001018 try:
1019 if pendingMap :
1020 parsedPending = json.loads( pendingMap )
1021 main.log.warn( json.dumps( parsedPending,
1022 sort_keys=True,
1023 indent=4,
1024 separators=( ',', ': ' ) ) )
1025 # TODO check something here?
1026 else:
1027 main.log.error( "pendingMap() returned None" )
1028 except ( ValueError, TypeError ):
1029 main.log.exception( "Error parsing pending map" )
1030 main.log.error( repr( pendingMap ) )
1031
1032 if not installedCheck:
1033 main.log.info( "Waiting 60 seconds to see if the state of " +
1034 "intents change" )
1035 time.sleep( 60 )
1036 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001037 intents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -07001038 intentStates = []
1039 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1040 count = 0
1041 # Iter through intents of a node
1042 try:
1043 for intent in json.loads( intents ):
1044 state = intent.get( 'state', None )
1045 if "INSTALLED" not in state:
1046 installedCheck = False
1047 intentId = intent.get( 'id', None )
1048 intentStates.append( ( intentId, state ) )
1049 except ( ValueError, TypeError ):
1050 main.log.exception( "Error parsing intents." )
1051 intentStates.sort()
1052 for i, s in intentStates:
1053 count += 1
1054 main.log.info( "%-6s%-15s%-15s" %
1055 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001056 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -07001057 try:
1058 missing = False
1059 if leaders:
1060 parsedLeaders = json.loads( leaders )
1061 main.log.warn( json.dumps( parsedLeaders,
1062 sort_keys=True,
1063 indent=4,
1064 separators=( ',', ': ' ) ) )
1065 # check for all intent partitions
1066 # check for election
1067 topics = []
1068 for i in range( 14 ):
1069 topics.append( "intent-partition-" + str( i ) )
1070 # FIXME: this should only be after we start the app
1071 topics.append( "org.onosproject.election" )
1072 main.log.debug( topics )
1073 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1074 for topic in topics:
1075 if topic not in ONOStopics:
1076 main.log.error( "Error: " + topic +
1077 " not in leaders" )
1078 missing = True
1079 else:
1080 main.log.error( "leaders() returned None" )
1081 except ( ValueError, TypeError ):
1082 main.log.exception( "Error parsing leaders" )
1083 main.log.error( repr( leaders ) )
1084 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001085 for i in main.activeNodes:
1086 node = main.CLIs[i]
1087 response = node.leaders( jsonFormat=False)
1088 main.log.warn( str( node.name ) + " leaders output: \n" +
1089 str( response ) )
1090
1091 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -07001092 try:
1093 if partitions :
1094 parsedPartitions = json.loads( partitions )
1095 main.log.warn( json.dumps( parsedPartitions,
1096 sort_keys=True,
1097 indent=4,
1098 separators=( ',', ': ' ) ) )
1099 # TODO check for a leader in all paritions
1100 # TODO check for consistency among nodes
1101 else:
1102 main.log.error( "partitions() returned None" )
1103 except ( ValueError, TypeError ):
1104 main.log.exception( "Error parsing partitions" )
1105 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001106 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -07001107 try:
1108 if pendingMap :
1109 parsedPending = json.loads( pendingMap )
1110 main.log.warn( json.dumps( parsedPending,
1111 sort_keys=True,
1112 indent=4,
1113 separators=( ',', ': ' ) ) )
1114 # TODO check something here?
1115 else:
1116 main.log.error( "pendingMap() returned None" )
1117 except ( ValueError, TypeError ):
1118 main.log.exception( "Error parsing pending map" )
1119 main.log.error( repr( pendingMap ) )
1120 # Print flowrules
Jon Halla440e872016-03-31 15:15:50 -07001121 node = main.activeNodes[0]
1122 main.log.debug( main.CLIs[node].flows( jsonFormat=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001123 main.step( "Wait a minute then ping again" )
1124 # the wait is above
1125 PingResult = main.TRUE
1126 for i in range( 8, 18 ):
1127 ping = main.Mininet1.pingHost( src="h" + str( i ),
1128 target="h" + str( i + 10 ) )
1129 PingResult = PingResult and ping
1130 if ping == main.FALSE:
1131 main.log.warn( "Ping failed between h" + str( i ) +
1132 " and h" + str( i + 10 ) )
1133 elif ping == main.TRUE:
1134 main.log.info( "Ping test passed!" )
1135 # Don't set PingResult or you'd override failures
1136 if PingResult == main.FALSE:
1137 main.log.error(
1138 "Intents have not been installed correctly, pings failed." )
1139 # TODO: pretty print
1140 main.log.warn( "ONOS1 intents: " )
1141 try:
Jon Halla440e872016-03-31 15:15:50 -07001142 tmpIntents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -07001143 main.log.warn( json.dumps( json.loads( tmpIntents ),
1144 sort_keys=True,
1145 indent=4,
1146 separators=( ',', ': ' ) ) )
1147 except ( ValueError, TypeError ):
1148 main.log.warn( repr( tmpIntents ) )
1149 utilities.assert_equals(
1150 expect=main.TRUE,
1151 actual=PingResult,
1152 onpass="Intents have been installed correctly and pings work",
1153 onfail="Intents have not been installed correctly, pings failed." )
1154
1155 def CASE5( self, main ):
1156 """
1157 Reading state of ONOS
1158 """
1159 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001160 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001161 assert main, "main not defined"
1162 assert utilities.assert_equals, "utilities.assert_equals not defined"
1163
1164 main.case( "Setting up and gathering data for current state" )
1165 # The general idea for this test case is to pull the state of
1166 # ( intents,flows, topology,... ) from each ONOS node
1167 # We can then compare them with each other and also with past states
1168
1169 main.step( "Check that each switch has a master" )
1170 global mastershipState
1171 mastershipState = '[]'
1172
1173 # Assert that each device has a master
Jon Halla440e872016-03-31 15:15:50 -07001174 rolesNotNull = main.TRUE
1175 threads = []
1176 for i in main.activeNodes:
1177 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1178 name="rolesNotNull-" + str( i ),
1179 args=[] )
1180 threads.append( t )
1181 t.start()
1182
1183 for t in threads:
1184 t.join()
1185 rolesNotNull = rolesNotNull and t.result
Jon Hall85794ff2015-07-08 14:12:30 -07001186 utilities.assert_equals(
1187 expect=main.TRUE,
1188 actual=rolesNotNull,
1189 onpass="Each device has a master",
1190 onfail="Some devices don't have a master assigned" )
1191
1192 main.step( "Get the Mastership of each switch" )
1193 ONOS1Mastership = main.ONOScli1.roles()
1194 # TODO: Make this a meaningful check
1195 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1196 main.log.error( "Error in getting ONOS roles" )
1197 main.log.warn(
1198 "ONOS1 mastership response: " +
1199 repr( ONOS1Mastership ) )
1200 consistentMastership = main.FALSE
1201 else:
1202 mastershipState = ONOS1Mastership
1203 consistentMastership = main.TRUE
1204
1205 main.step( "Get the intents from each controller" )
1206 global intentState
1207 intentState = []
1208 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1209 intentCheck = main.FALSE
1210 if "Error" in ONOS1Intents or not ONOS1Intents:
1211 main.log.error( "Error in getting ONOS intents" )
1212 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1213 else:
1214 intentCheck = main.TRUE
1215
1216 main.step( "Get the flows from each controller" )
1217 global flowState
1218 flowState = []
1219 flowCheck = main.FALSE
1220 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
1221 if "Error" in ONOS1Flows or not ONOS1Flows:
1222 main.log.error( "Error in getting ONOS flows" )
1223 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
1224 else:
1225 # TODO: Do a better check, maybe compare flows on switches?
1226 flowState = ONOS1Flows
1227 flowCheck = main.TRUE
1228
1229 main.step( "Get the OF Table entries" )
1230 global flows
1231 flows = []
1232 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001233 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001234 if flowCheck == main.FALSE:
1235 for table in flows:
1236 main.log.warn( table )
1237 # TODO: Compare switch flow tables with ONOS flow tables
1238
1239 main.step( "Collecting topology information from ONOS" )
1240 devices = []
1241 devices.append( main.ONOScli1.devices() )
1242 hosts = []
1243 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1244 ports = []
1245 ports.append( main.ONOScli1.ports() )
1246 links = []
1247 links.append( main.ONOScli1.links() )
1248 clusters = []
1249 clusters.append( main.ONOScli1.clusters() )
1250
1251 main.step( "Each host has an IP address" )
1252 ipResult = main.TRUE
1253 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001254 controllerStr = str( main.activeNodes[controller] + 1 )
1255 if hosts[ controller ]:
1256 for host in hosts[ controller ]:
1257 if not host.get( 'ipAddresses', [ ] ):
1258 main.log.error( "Error with host ips on controller" +
1259 controllerStr + ": " + str( host ) )
1260 ipResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07001261 utilities.assert_equals(
1262 expect=main.TRUE,
1263 actual=ipResult,
1264 onpass="The ips of the hosts aren't empty",
1265 onfail="The ip of at least one host is missing" )
1266
1267 # there should always only be one cluster
1268 main.step( "There is only one dataplane cluster" )
1269 try:
1270 numClusters = len( json.loads( clusters[ 0 ] ) )
1271 except ( ValueError, TypeError ):
1272 main.log.exception( "Error parsing clusters[0]: " +
1273 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001274 numClusters = "ERROR"
Jon Hall85794ff2015-07-08 14:12:30 -07001275 clusterResults = main.FALSE
1276 if numClusters == 1:
1277 clusterResults = main.TRUE
1278 utilities.assert_equals(
1279 expect=1,
1280 actual=numClusters,
1281 onpass="ONOS shows 1 SCC",
1282 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1283
1284 main.step( "Comparing ONOS topology to MN" )
1285 devicesResults = main.TRUE
1286 linksResults = main.TRUE
1287 hostsResults = main.TRUE
1288 mnSwitches = main.Mininet1.getSwitches()
1289 mnLinks = main.Mininet1.getLinks()
1290 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001291 for controller in main.activeNodes:
1292 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07001293 if devices[ controller ] and ports[ controller ] and\
1294 "Error" not in devices[ controller ] and\
1295 "Error" not in ports[ controller ]:
Jon Hall6e709752016-02-01 13:38:46 -08001296 currentDevicesResult = main.Mininet1.compareSwitches(
1297 mnSwitches,
1298 json.loads( devices[ controller ] ),
1299 json.loads( ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001300 else:
1301 currentDevicesResult = main.FALSE
1302 utilities.assert_equals( expect=main.TRUE,
1303 actual=currentDevicesResult,
1304 onpass="ONOS" + controllerStr +
1305 " Switches view is correct",
1306 onfail="ONOS" + controllerStr +
1307 " Switches view is incorrect" )
1308 if links[ controller ] and "Error" not in links[ controller ]:
1309 currentLinksResult = main.Mininet1.compareLinks(
1310 mnSwitches, mnLinks,
1311 json.loads( links[ controller ] ) )
1312 else:
1313 currentLinksResult = main.FALSE
1314 utilities.assert_equals( expect=main.TRUE,
1315 actual=currentLinksResult,
1316 onpass="ONOS" + controllerStr +
1317 " links view is correct",
1318 onfail="ONOS" + controllerStr +
1319 " links view is incorrect" )
1320
Jon Halla440e872016-03-31 15:15:50 -07001321 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall85794ff2015-07-08 14:12:30 -07001322 currentHostsResult = main.Mininet1.compareHosts(
1323 mnHosts,
1324 hosts[ controller ] )
1325 else:
1326 currentHostsResult = main.FALSE
1327 utilities.assert_equals( expect=main.TRUE,
1328 actual=currentHostsResult,
1329 onpass="ONOS" + controllerStr +
1330 " hosts exist in Mininet",
1331 onfail="ONOS" + controllerStr +
1332 " hosts don't match Mininet" )
1333
1334 devicesResults = devicesResults and currentDevicesResult
1335 linksResults = linksResults and currentLinksResult
1336 hostsResults = hostsResults and currentHostsResult
1337
1338 main.step( "Device information is correct" )
1339 utilities.assert_equals(
1340 expect=main.TRUE,
1341 actual=devicesResults,
1342 onpass="Device information is correct",
1343 onfail="Device information is incorrect" )
1344
1345 main.step( "Links are correct" )
1346 utilities.assert_equals(
1347 expect=main.TRUE,
1348 actual=linksResults,
1349 onpass="Link are correct",
1350 onfail="Links are incorrect" )
1351
1352 main.step( "Hosts are correct" )
1353 utilities.assert_equals(
1354 expect=main.TRUE,
1355 actual=hostsResults,
1356 onpass="Hosts are correct",
1357 onfail="Hosts are incorrect" )
1358
1359 def CASE6( self, main ):
1360 """
1361 The Failure case.
1362 """
1363 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001364 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001365 assert main, "main not defined"
1366 assert utilities.assert_equals, "utilities.assert_equals not defined"
1367
1368 # Reset non-persistent variables
1369 try:
1370 iCounterValue = 0
1371 except NameError:
1372 main.log.error( "iCounterValue not defined, setting to 0" )
1373 iCounterValue = 0
1374
1375 main.case( "Restart ONOS node" )
Jon Hall783bbf92015-07-23 14:33:19 -07001376 main.caseExplanation = "Killing ONOS process and restart cli " +\
Jon Hall85794ff2015-07-08 14:12:30 -07001377 "sessions once onos is up."
Jon Hall96091e62015-09-21 17:34:17 -07001378
1379 main.step( "Checking ONOS Logs for errors" )
1380 for node in main.nodes:
1381 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1382 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1383
Jon Hall85794ff2015-07-08 14:12:30 -07001384 main.step( "Killing ONOS processes" )
Jon Halle1a3b752015-07-22 13:02:46 -07001385 killResult = main.ONOSbench.onosKill( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001386 start = time.time()
1387 utilities.assert_equals( expect=main.TRUE, actual=killResult,
1388 onpass="ONOS Killed",
1389 onfail="Error killing ONOS" )
1390
1391 main.step( "Checking if ONOS is up yet" )
1392 count = 0
1393 while count < 10:
Jon Halle1a3b752015-07-22 13:02:46 -07001394 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001395 if onos1Isup == main.TRUE:
1396 elapsed = time.time() - start
1397 break
1398 else:
1399 count = count + 1
1400 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
1401 onpass="ONOS is back up",
1402 onfail="ONOS failed to start" )
1403
1404 main.log.step( "Starting ONOS CLI sessions" )
Jon Halle1a3b752015-07-22 13:02:46 -07001405 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001406 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1407 onpass="ONOS cli startup successful",
1408 onfail="ONOS cli startup failed" )
1409
1410 if elapsed:
1411 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
1412 str( elapsed ) )
1413 main.restartTime = elapsed
1414 else:
1415 main.restartTime = -1
1416 time.sleep( 5 )
1417 # rerun on election apps
1418 main.ONOScli1.electionTestRun()
1419
1420 def CASE7( self, main ):
1421 """
1422 Check state after ONOS failure
1423 """
1424 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001425 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001426 assert main, "main not defined"
1427 assert utilities.assert_equals, "utilities.assert_equals not defined"
1428 main.case( "Running ONOS Constant State Tests" )
Jon Hall6e709752016-02-01 13:38:46 -08001429
Jon Hall85794ff2015-07-08 14:12:30 -07001430 main.step( "Check that each switch has a master" )
1431 # Assert that each device has a master
Jon Halla440e872016-03-31 15:15:50 -07001432 rolesNotNull = main.TRUE
1433 threads = []
1434 for i in main.activeNodes:
1435 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1436 name="rolesNotNull-" + str( i ),
1437 args=[ ] )
1438 threads.append( t )
1439 t.start()
1440
1441 for t in threads:
1442 t.join()
1443 rolesNotNull = rolesNotNull and t.result
Jon Hall85794ff2015-07-08 14:12:30 -07001444 utilities.assert_equals(
1445 expect=main.TRUE,
1446 actual=rolesNotNull,
1447 onpass="Each device has a master",
1448 onfail="Some devices don't have a master assigned" )
1449
1450 main.step( "Check if switch roles are consistent across all nodes" )
1451 ONOS1Mastership = main.ONOScli1.roles()
1452 # FIXME: Refactor this whole case for single instance
1453 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1454 main.log.error( "Error in getting ONOS mastership" )
1455 main.log.warn( "ONOS1 mastership response: " +
1456 repr( ONOS1Mastership ) )
1457 consistentMastership = main.FALSE
1458 else:
1459 consistentMastership = main.TRUE
1460 utilities.assert_equals(
1461 expect=main.TRUE,
1462 actual=consistentMastership,
1463 onpass="Switch roles are consistent across all ONOS nodes",
1464 onfail="ONOS nodes have different views of switch roles" )
1465
1466 description2 = "Compare switch roles from before failure"
1467 main.step( description2 )
1468
1469 currentJson = json.loads( ONOS1Mastership )
1470 oldJson = json.loads( mastershipState )
1471 mastershipCheck = main.TRUE
1472 for i in range( 1, 29 ):
1473 switchDPID = str(
1474 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1475
1476 current = [ switch[ 'master' ] for switch in currentJson
1477 if switchDPID in switch[ 'id' ] ]
1478 old = [ switch[ 'master' ] for switch in oldJson
1479 if switchDPID in switch[ 'id' ] ]
1480 if current == old:
1481 mastershipCheck = mastershipCheck and main.TRUE
1482 else:
1483 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1484 mastershipCheck = main.FALSE
1485 utilities.assert_equals(
1486 expect=main.TRUE,
1487 actual=mastershipCheck,
1488 onpass="Mastership of Switches was not changed",
1489 onfail="Mastership of some switches changed" )
1490 mastershipCheck = mastershipCheck and consistentMastership
1491
1492 main.step( "Get the intents and compare across all nodes" )
1493 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1494 intentCheck = main.FALSE
1495 if "Error" in ONOS1Intents or not ONOS1Intents:
1496 main.log.error( "Error in getting ONOS intents" )
1497 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1498 else:
1499 intentCheck = main.TRUE
1500 utilities.assert_equals(
1501 expect=main.TRUE,
1502 actual=intentCheck,
1503 onpass="Intents are consistent across all ONOS nodes",
1504 onfail="ONOS nodes have different views of intents" )
1505 # Print the intent states
1506 intents = []
1507 intents.append( ONOS1Intents )
1508 intentStates = []
1509 for node in intents: # Iter through ONOS nodes
1510 nodeStates = []
1511 # Iter through intents of a node
1512 for intent in json.loads( node ):
1513 nodeStates.append( intent[ 'state' ] )
1514 intentStates.append( nodeStates )
1515 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1516 main.log.info( dict( out ) )
1517
1518 # NOTE: Store has no durability, so intents are lost across system
1519 # restarts
1520 """
1521 main.step( "Compare current intents with intents before the failure" )
1522 # NOTE: this requires case 5 to pass for intentState to be set.
1523 # maybe we should stop the test if that fails?
1524 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07001525 try:
1526 intentState
1527 except NameError:
1528 main.log.warn( "No previous intent state was saved" )
1529 else:
1530 if intentState and intentState == ONOSIntents[ 0 ]:
1531 sameIntents = main.TRUE
1532 main.log.info( "Intents are consistent with before failure" )
1533 # TODO: possibly the states have changed? we may need to figure out
1534 # what the acceptable states are
1535 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1536 sameIntents = main.TRUE
1537 try:
1538 before = json.loads( intentState )
1539 after = json.loads( ONOSIntents[ 0 ] )
1540 for intent in before:
1541 if intent not in after:
1542 sameIntents = main.FALSE
1543 main.log.debug( "Intent is not currently in ONOS " +
1544 "(at least in the same form):" )
1545 main.log.debug( json.dumps( intent ) )
1546 except ( ValueError, TypeError ):
1547 main.log.exception( "Exception printing intents" )
1548 main.log.debug( repr( ONOSIntents[0] ) )
1549 main.log.debug( repr( intentState ) )
1550 if sameIntents == main.FALSE:
1551 try:
1552 main.log.debug( "ONOS intents before: " )
1553 main.log.debug( json.dumps( json.loads( intentState ),
1554 sort_keys=True, indent=4,
1555 separators=( ',', ': ' ) ) )
1556 main.log.debug( "Current ONOS intents: " )
1557 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1558 sort_keys=True, indent=4,
1559 separators=( ',', ': ' ) ) )
1560 except ( ValueError, TypeError ):
1561 main.log.exception( "Exception printing intents" )
1562 main.log.debug( repr( ONOSIntents[0] ) )
1563 main.log.debug( repr( intentState ) )
1564 utilities.assert_equals(
1565 expect=main.TRUE,
1566 actual=sameIntents,
1567 onpass="Intents are consistent with before failure",
1568 onfail="The Intents changed during failure" )
Jon Hall85794ff2015-07-08 14:12:30 -07001569 intentCheck = intentCheck and sameIntents
1570 """
1571 main.step( "Get the OF Table entries and compare to before " +
1572 "component failure" )
1573 FlowTables = main.TRUE
Jon Hall85794ff2015-07-08 14:12:30 -07001574 for i in range( 28 ):
1575 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08001576 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07001577 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
1578 FlowTables = FlowTables and curSwitch
1579 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08001580 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001581 utilities.assert_equals(
1582 expect=main.TRUE,
1583 actual=FlowTables,
1584 onpass="No changes were found in the flow tables",
1585 onfail="Changes were found in the flow tables" )
1586
1587 main.step( "Leadership Election is still functional" )
1588 # Test of LeadershipElection
1589
Jon Halla440e872016-03-31 15:15:50 -07001590 leader = main.nodes[ main.activeNodes[ 0 ] ].ip_address
Jon Hall85794ff2015-07-08 14:12:30 -07001591 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001592 for controller in range( 1, main.numCtrls + 1 ):
Jon Hall85794ff2015-07-08 14:12:30 -07001593 # loop through ONOScli handlers
1594 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
1595 leaderN = node.electionTestLeader()
1596 # verify leader is ONOS1
1597 # NOTE even though we restarted ONOS, it is the only one so onos 1
1598 # must be leader
1599 if leaderN == leader:
1600 # all is well
1601 pass
1602 elif leaderN == main.FALSE:
1603 # error in response
1604 main.log.error( "Something is wrong with " +
1605 "electionTestLeader function, check the" +
1606 " error logs" )
1607 leaderResult = main.FALSE
1608 elif leader != leaderN:
1609 leaderResult = main.FALSE
1610 main.log.error( "ONOS" + str( controller ) + " sees " +
1611 str( leaderN ) +
1612 " as the leader of the election app. " +
1613 "Leader should be " + str( leader ) )
1614 utilities.assert_equals(
1615 expect=main.TRUE,
1616 actual=leaderResult,
1617 onpass="Leadership election passed",
1618 onfail="Something went wrong with Leadership election" )
1619
1620 def CASE8( self, main ):
1621 """
1622 Compare topo
1623 """
1624 import json
1625 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001626 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001627 assert main, "main not defined"
1628 assert utilities.assert_equals, "utilities.assert_equals not defined"
1629
1630 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07001631 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall85794ff2015-07-08 14:12:30 -07001632 " and ONOS"
Jon Hall85794ff2015-07-08 14:12:30 -07001633 topoResult = main.FALSE
1634 elapsed = 0
1635 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08001636 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall85794ff2015-07-08 14:12:30 -07001637 startTime = time.time()
1638 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08001639 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07001640 devicesResults = main.TRUE
1641 linksResults = main.TRUE
1642 hostsResults = main.TRUE
1643 hostAttachmentResults = True
Jon Hall85794ff2015-07-08 14:12:30 -07001644 count += 1
1645 cliStart = time.time()
1646 devices = []
1647 devices.append( main.ONOScli1.devices() )
1648 hosts = []
1649 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1650 ipResult = main.TRUE
1651 for controller in range( 0, len( hosts ) ):
1652 controllerStr = str( controller + 1 )
1653 for host in hosts[ controller ]:
1654 if host is None or host.get( 'ipAddresses', [] ) == []:
1655 main.log.error(
1656 "DEBUG:Error with host ips on controller" +
1657 controllerStr + ": " + str( host ) )
1658 ipResult = main.FALSE
1659 ports = []
1660 ports.append( main.ONOScli1.ports() )
1661 links = []
1662 links.append( main.ONOScli1.links() )
1663 clusters = []
1664 clusters.append( main.ONOScli1.clusters() )
1665
1666 elapsed = time.time() - startTime
1667 cliTime = time.time() - cliStart
1668 print "CLI time: " + str( cliTime )
1669
1670 mnSwitches = main.Mininet1.getSwitches()
1671 mnLinks = main.Mininet1.getLinks()
1672 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001673 for controller in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07001674 controllerStr = str( controller + 1 )
1675 if devices[ controller ] and ports[ controller ] and\
1676 "Error" not in devices[ controller ] and\
1677 "Error" not in ports[ controller ]:
1678
Jon Hallc6793552016-01-19 14:18:37 -08001679 try:
1680 currentDevicesResult = main.Mininet1.compareSwitches(
1681 mnSwitches,
1682 json.loads( devices[ controller ] ),
1683 json.loads( ports[ controller ] ) )
1684 except ( TypeError, ValueError ) as e:
1685 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
1686 devices[ controller ], ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001687 else:
1688 currentDevicesResult = main.FALSE
1689 utilities.assert_equals( expect=main.TRUE,
1690 actual=currentDevicesResult,
1691 onpass="ONOS" + controllerStr +
1692 " Switches view is correct",
1693 onfail="ONOS" + controllerStr +
1694 " Switches view is incorrect" )
1695
1696 if links[ controller ] and "Error" not in links[ controller ]:
1697 currentLinksResult = main.Mininet1.compareLinks(
1698 mnSwitches, mnLinks,
1699 json.loads( links[ controller ] ) )
1700 else:
1701 currentLinksResult = main.FALSE
1702 utilities.assert_equals( expect=main.TRUE,
1703 actual=currentLinksResult,
1704 onpass="ONOS" + controllerStr +
1705 " links view is correct",
1706 onfail="ONOS" + controllerStr +
1707 " links view is incorrect" )
1708
1709 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1710 currentHostsResult = main.Mininet1.compareHosts(
1711 mnHosts,
1712 hosts[ controller ] )
1713 else:
1714 currentHostsResult = main.FALSE
1715 utilities.assert_equals( expect=main.TRUE,
1716 actual=currentHostsResult,
1717 onpass="ONOS" + controllerStr +
1718 " hosts exist in Mininet",
1719 onfail="ONOS" + controllerStr +
1720 " hosts don't match Mininet" )
1721 # CHECKING HOST ATTACHMENT POINTS
1722 hostAttachment = True
1723 zeroHosts = False
1724 # FIXME: topo-HA/obelisk specific mappings:
1725 # key is mac and value is dpid
1726 mappings = {}
1727 for i in range( 1, 29 ): # hosts 1 through 28
1728 # set up correct variables:
1729 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
1730 if i == 1:
1731 deviceId = "1000".zfill(16)
1732 elif i == 2:
1733 deviceId = "2000".zfill(16)
1734 elif i == 3:
1735 deviceId = "3000".zfill(16)
1736 elif i == 4:
1737 deviceId = "3004".zfill(16)
1738 elif i == 5:
1739 deviceId = "5000".zfill(16)
1740 elif i == 6:
1741 deviceId = "6000".zfill(16)
1742 elif i == 7:
1743 deviceId = "6007".zfill(16)
1744 elif i >= 8 and i <= 17:
1745 dpid = '3' + str( i ).zfill( 3 )
1746 deviceId = dpid.zfill(16)
1747 elif i >= 18 and i <= 27:
1748 dpid = '6' + str( i ).zfill( 3 )
1749 deviceId = dpid.zfill(16)
1750 elif i == 28:
1751 deviceId = "2800".zfill(16)
1752 mappings[ macId ] = deviceId
1753 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1754 if hosts[ controller ] == []:
1755 main.log.warn( "There are no hosts discovered" )
1756 zeroHosts = True
1757 else:
1758 for host in hosts[ controller ]:
1759 mac = None
1760 location = None
1761 device = None
1762 port = None
1763 try:
1764 mac = host.get( 'mac' )
1765 assert mac, "mac field could not be found for this host object"
1766
1767 location = host.get( 'location' )
1768 assert location, "location field could not be found for this host object"
1769
1770 # Trim the protocol identifier off deviceId
1771 device = str( location.get( 'elementId' ) ).split(':')[1]
1772 assert device, "elementId field could not be found for this host location object"
1773
1774 port = location.get( 'port' )
1775 assert port, "port field could not be found for this host location object"
1776
1777 # Now check if this matches where they should be
1778 if mac and device and port:
1779 if str( port ) != "1":
1780 main.log.error( "The attachment port is incorrect for " +
1781 "host " + str( mac ) +
1782 ". Expected: 1 Actual: " + str( port) )
1783 hostAttachment = False
1784 if device != mappings[ str( mac ) ]:
1785 main.log.error( "The attachment device is incorrect for " +
1786 "host " + str( mac ) +
1787 ". Expected: " + mappings[ str( mac ) ] +
1788 " Actual: " + device )
1789 hostAttachment = False
1790 else:
1791 hostAttachment = False
1792 except AssertionError:
1793 main.log.exception( "Json object not as expected" )
1794 main.log.error( repr( host ) )
1795 hostAttachment = False
1796 else:
1797 main.log.error( "No hosts json output or \"Error\"" +
1798 " in output. hosts = " +
1799 repr( hosts[ controller ] ) )
1800 if zeroHosts is False:
1801 hostAttachment = True
1802
Jon Hall85794ff2015-07-08 14:12:30 -07001803 devicesResults = devicesResults and currentDevicesResult
1804 linksResults = linksResults and currentLinksResult
1805 hostsResults = hostsResults and currentHostsResult
1806 hostAttachmentResults = hostAttachmentResults and\
1807 hostAttachment
1808
Jon Halla440e872016-03-31 15:15:50 -07001809 # "consistent" results don't make sense for single instance
Jon Hall85794ff2015-07-08 14:12:30 -07001810 # there should always only be one cluster
Jon Hall85794ff2015-07-08 14:12:30 -07001811 clusterResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07001812 try:
1813 numClusters = len( json.loads( clusters[ 0 ] ) )
1814 except ( ValueError, TypeError ):
1815 main.log.exception( "Error parsing clusters[0]: " +
1816 repr( clusters[0] ) )
1817 numClusters = "ERROR"
1818 clusterResults = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07001819 if numClusters == 1:
1820 clusterResults = main.TRUE
1821 utilities.assert_equals(
1822 expect=1,
1823 actual=numClusters,
1824 onpass="ONOS shows 1 SCC",
1825 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1826
1827 topoResult = ( devicesResults and linksResults
1828 and hostsResults and ipResult and clusterResults and
1829 hostAttachmentResults )
1830
1831 topoResult = topoResult and int( count <= 2 )
1832 note = "note it takes about " + str( int( cliTime ) ) + \
1833 " seconds for the test to make all the cli calls to fetch " +\
1834 "the topology from each ONOS instance"
1835 main.log.info(
1836 "Very crass estimate for topology discovery/convergence( " +
1837 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1838 str( count ) + " tries" )
1839 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1840 onpass="Topology Check Test successful",
1841 onfail="Topology Check Test NOT successful" )
Jon Hall41d39f12016-04-11 22:54:35 -07001842 main.step( "Checking ONOS nodes" )
1843 nodeResults = utilities.retry( main.HA.nodesCheck,
1844 False,
1845 args=[main.activeNodes],
1846 attempts=5 )
1847
1848 utilities.assert_equals( expect=True, actual=nodeResults,
1849 onpass="Nodes check successful",
1850 onfail="Nodes check NOT successful" )
1851 if not nodeResults:
1852 for i in main.activeNodes:
1853 main.log.debug( "{} components not ACTIVE: \n{}".format(
1854 main.CLIs[i].name,
1855 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001856
1857 def CASE9( self, main ):
1858 """
1859 Link s3-s28 down
1860 """
1861 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001862 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001863 assert main, "main not defined"
1864 assert utilities.assert_equals, "utilities.assert_equals not defined"
1865 # NOTE: You should probably run a topology check after this
1866
1867 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1868
1869 description = "Turn off a link to ensure that Link Discovery " +\
1870 "is working properly"
1871 main.case( description )
1872
1873 main.step( "Kill Link between s3 and s28" )
1874 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
1875 main.log.info( "Waiting " + str( linkSleep ) +
1876 " seconds for link down to be discovered" )
1877 time.sleep( linkSleep )
1878 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
1879 onpass="Link down successful",
1880 onfail="Failed to bring link down" )
1881 # TODO do some sort of check here
1882
1883 def CASE10( self, main ):
1884 """
1885 Link s3-s28 up
1886 """
1887 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001888 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001889 assert main, "main not defined"
1890 assert utilities.assert_equals, "utilities.assert_equals not defined"
1891 # NOTE: You should probably run a topology check after this
1892
1893 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1894
1895 description = "Restore a link to ensure that Link Discovery is " + \
1896 "working properly"
1897 main.case( description )
1898
1899 main.step( "Bring link between s3 and s28 back up" )
1900 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
1901 main.log.info( "Waiting " + str( linkSleep ) +
1902 " seconds for link up to be discovered" )
1903 time.sleep( linkSleep )
1904 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
1905 onpass="Link up successful",
1906 onfail="Failed to bring link up" )
1907 # TODO do some sort of check here
1908
1909 def CASE11( self, main ):
1910 """
1911 Switch Down
1912 """
1913 # NOTE: You should probably run a topology check after this
1914 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001915 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001916 assert main, "main not defined"
1917 assert utilities.assert_equals, "utilities.assert_equals not defined"
1918
1919 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1920
1921 description = "Killing a switch to ensure it is discovered correctly"
Jon Halla440e872016-03-31 15:15:50 -07001922 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall85794ff2015-07-08 14:12:30 -07001923 main.case( description )
1924 switch = main.params[ 'kill' ][ 'switch' ]
1925 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1926
1927 # TODO: Make this switch parameterizable
1928 main.step( "Kill " + switch )
1929 main.log.info( "Deleting " + switch )
1930 main.Mininet1.delSwitch( switch )
1931 main.log.info( "Waiting " + str( switchSleep ) +
1932 " seconds for switch down to be discovered" )
1933 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07001934 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall85794ff2015-07-08 14:12:30 -07001935 # Peek at the deleted switch
1936 main.log.warn( str( device ) )
1937 result = main.FALSE
1938 if device and device[ 'available' ] is False:
1939 result = main.TRUE
1940 utilities.assert_equals( expect=main.TRUE, actual=result,
1941 onpass="Kill switch successful",
1942 onfail="Failed to kill switch?" )
1943
1944 def CASE12( self, main ):
1945 """
1946 Switch Up
1947 """
1948 # NOTE: You should probably run a topology check after this
1949 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001950 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001951 assert main, "main not defined"
1952 assert utilities.assert_equals, "utilities.assert_equals not defined"
1953
1954 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1955 switch = main.params[ 'kill' ][ 'switch' ]
1956 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1957 links = main.params[ 'kill' ][ 'links' ].split()
Jon Halla440e872016-03-31 15:15:50 -07001958 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall85794ff2015-07-08 14:12:30 -07001959 description = "Adding a switch to ensure it is discovered correctly"
1960 main.case( description )
1961
1962 main.step( "Add back " + switch )
1963 main.Mininet1.addSwitch( switch, dpid=switchDPID )
1964 for peer in links:
1965 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07001966 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall85794ff2015-07-08 14:12:30 -07001967 main.Mininet1.assignSwController( sw=switch, ip=ipList )
1968 main.log.info( "Waiting " + str( switchSleep ) +
1969 " seconds for switch up to be discovered" )
1970 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07001971 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall85794ff2015-07-08 14:12:30 -07001972 # Peek at the deleted switch
1973 main.log.warn( str( device ) )
1974 result = main.FALSE
1975 if device and device[ 'available' ]:
1976 result = main.TRUE
1977 utilities.assert_equals( expect=main.TRUE, actual=result,
1978 onpass="add switch successful",
1979 onfail="Failed to add switch?" )
1980
1981 def CASE13( self, main ):
1982 """
1983 Clean up
1984 """
1985 import os
1986 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001987 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001988 assert main, "main not defined"
1989 assert utilities.assert_equals, "utilities.assert_equals not defined"
1990 # printing colors to terminal
1991 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
1992 'blue': '\033[94m', 'green': '\033[92m',
1993 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
1994 main.case( "Test Cleanup" )
1995 main.step( "Killing tcpdumps" )
1996 main.Mininet2.stopTcpdump()
1997
1998 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07001999 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall85794ff2015-07-08 14:12:30 -07002000 main.step( "Copying MN pcap and ONOS log files to test station" )
2001 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2002 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002003 # NOTE: MN Pcap file is being saved to logdir.
2004 # We scp this file as MN and TestON aren't necessarily the same vm
2005
2006 # FIXME: To be replaced with a Jenkin's post script
Jon Hall85794ff2015-07-08 14:12:30 -07002007 # TODO: Load these from params
2008 # NOTE: must end in /
2009 logFolder = "/opt/onos/log/"
2010 logFiles = [ "karaf.log", "karaf.log.1" ]
2011 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07002012 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07002013 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002014 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002015 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2016 logFolder + f, dstName )
Jon Hall85794ff2015-07-08 14:12:30 -07002017 # std*.log's
2018 # NOTE: must end in /
2019 logFolder = "/opt/onos/var/"
2020 logFiles = [ "stderr.log", "stdout.log" ]
2021 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07002022 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07002023 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002024 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002025 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2026 logFolder + f, dstName )
2027 else:
2028 main.log.debug( "skipping saving log files" )
Jon Hall85794ff2015-07-08 14:12:30 -07002029
Jon Hall85794ff2015-07-08 14:12:30 -07002030 main.step( "Stopping Mininet" )
2031 mnResult = main.Mininet1.stopNet()
2032 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2033 onpass="Mininet stopped",
2034 onfail="MN cleanup NOT successful" )
2035
2036 main.step( "Checking ONOS Logs for errors" )
Jon Hall96091e62015-09-21 17:34:17 -07002037 for node in main.nodes:
2038 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2039 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall85794ff2015-07-08 14:12:30 -07002040
2041 try:
2042 timerLog = open( main.logdir + "/Timers.csv", 'w')
2043 # Overwrite with empty line and close
2044 labels = "Gossip Intents, Restart"
2045 data = str( gossipTime ) + ", " + str( main.restartTime )
2046 timerLog.write( labels + "\n" + data )
2047 timerLog.close()
2048 except NameError, e:
2049 main.log.exception(e)
2050
2051 def CASE14( self, main ):
2052 """
2053 start election app on all onos nodes
2054 """
Jon Halle1a3b752015-07-22 13:02:46 -07002055 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002056 assert main, "main not defined"
2057 assert utilities.assert_equals, "utilities.assert_equals not defined"
2058
2059 main.case("Start Leadership Election app")
2060 main.step( "Install leadership election app" )
Jon Halla440e872016-03-31 15:15:50 -07002061 onosCli = main.CLIs[ main.activeNodes[0] ]
2062 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall85794ff2015-07-08 14:12:30 -07002063 utilities.assert_equals(
2064 expect=main.TRUE,
2065 actual=appResult,
2066 onpass="Election app installed",
2067 onfail="Something went wrong with installing Leadership election" )
2068
2069 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002070 for i in main.activeNodes:
2071 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002072 time.sleep(5)
2073 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2074 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall85794ff2015-07-08 14:12:30 -07002075 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002076 expect=True,
2077 actual=sameResult,
2078 onpass="All nodes see the same leaderboards",
2079 onfail="Inconsistent leaderboards" )
2080
2081 if sameResult:
2082 leader = leaders[ 0 ][ 0 ]
2083 if main.nodes[main.activeNodes[0]].ip_address in leader:
2084 correctLeader = True
2085 else:
2086 correctLeader = False
2087 main.step( "First node was elected leader" )
2088 utilities.assert_equals(
2089 expect=True,
2090 actual=correctLeader,
2091 onpass="Correct leader was elected",
2092 onfail="Incorrect leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07002093
2094 def CASE15( self, main ):
2095 """
2096 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002097 15.1 Run election on each node
2098 15.2 Check that each node has the same leaders and candidates
2099 15.3 Find current leader and withdraw
2100 15.4 Check that a new node was elected leader
2101 15.5 Check that that new leader was the candidate of old leader
2102 15.6 Run for election on old leader
2103 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2104 15.8 Make sure that the old leader was added to the candidate list
2105
2106 old and new variable prefixes refer to data from before vs after
2107 withdrawl and later before withdrawl vs after re-election
Jon Hall85794ff2015-07-08 14:12:30 -07002108 """
acsmars71adceb2015-08-31 15:09:26 -07002109 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002110 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002111 assert main, "main not defined"
2112 assert utilities.assert_equals, "utilities.assert_equals not defined"
acsmars71adceb2015-08-31 15:09:26 -07002113 assert main.CLIs, "main.CLIs not defined"
2114 assert main.nodes, "main.nodes not defined"
2115
Jon Hall85794ff2015-07-08 14:12:30 -07002116 description = "Check that Leadership Election is still functional"
2117 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002118 # NOTE: Need to re-run after restarts since being a canidate is not persistant
acsmars2c2fcdd2015-08-25 17:14:13 -07002119
Jon Halla440e872016-03-31 15:15:50 -07002120 oldLeaders = [] # list of lists of each nodes' candidates before
2121 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002122 oldLeader = '' # the old leader from oldLeaders, None if not same
2123 newLeader = '' # the new leaders fron newLoeaders, None if not same
2124 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2125 expectNoLeader = False # True when there is only one leader
2126 if main.numCtrls == 1:
2127 expectNoLeader = True
acsmars2c2fcdd2015-08-25 17:14:13 -07002128
acsmars71adceb2015-08-31 15:09:26 -07002129 main.step( "Run for election on each node" )
2130 electionResult = main.TRUE
2131
Jon Halla440e872016-03-31 15:15:50 -07002132 for i in main.activeNodes: # run test election on each node
2133 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002134 electionResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002135 utilities.assert_equals(
2136 expect=main.TRUE,
2137 actual=electionResult,
2138 onpass="All nodes successfully ran for leadership",
2139 onfail="At least one node failed to run for leadership" )
2140
acsmars3a72bde2015-09-02 14:16:22 -07002141 if electionResult == main.FALSE:
2142 main.log.error(
2143 "Skipping Test Case because Election Test App isn't loaded" )
2144 main.skipCase()
2145
acsmars71adceb2015-08-31 15:09:26 -07002146 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002147 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002148 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002149 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002150 if sameResult:
2151 oldLeader = oldLeaders[ 0 ][ 0 ]
2152 main.log.warn( oldLeader )
Jon Hall85794ff2015-07-08 14:12:30 -07002153 else:
Jon Halla440e872016-03-31 15:15:50 -07002154 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002155 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002156 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002157 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002158 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002159 onfail=failMessage )
2160
2161 main.step( "Find current leader and withdraw" )
2162 withdrawResult = main.TRUE
2163 # do some sanity checking on leader before using it
2164 if oldLeader is None:
2165 main.log.error( "Leadership isn't consistent." )
2166 withdrawResult = main.FALSE
2167 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002168 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002169 if oldLeader == main.nodes[ i ].ip_address:
2170 oldLeaderCLI = main.CLIs[ i ]
2171 break
2172 else: # FOR/ELSE statement
2173 main.log.error( "Leader election, could not find current leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07002174 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002175 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall85794ff2015-07-08 14:12:30 -07002176 utilities.assert_equals(
2177 expect=main.TRUE,
2178 actual=withdrawResult,
2179 onpass="Node was withdrawn from election",
2180 onfail="Node was not withdrawn from election" )
2181
acsmars71adceb2015-08-31 15:09:26 -07002182 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002183 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07002184 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002185 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002186 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002187 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002188 if newLeaders[ 0 ][ 0 ] == 'none':
2189 main.log.error( "No leader was elected on at least 1 node" )
2190 if not expectNoLeader:
2191 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002192 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002193
2194 # Check that the new leader is not the older leader, which was withdrawn
2195 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002196 newLeaderResult = False
2197 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07002198 " as the current leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07002199 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002200 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002201 actual=newLeaderResult,
2202 onpass="Leadership election passed",
2203 onfail="Something went wrong with Leadership election" )
Jon Hall85794ff2015-07-08 14:12:30 -07002204
Jon Halla440e872016-03-31 15:15:50 -07002205 main.step( "Check that that new leader was the candidate of old leader" )
2206 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07002207 correctCandidateResult = main.TRUE
2208 if expectNoLeader:
2209 if newLeader == 'none':
2210 main.log.info( "No leader expected. None found. Pass" )
2211 correctCandidateResult = main.TRUE
2212 else:
2213 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2214 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002215 elif len( oldLeaders[0] ) >= 3:
2216 if newLeader == oldLeaders[ 0 ][ 2 ]:
2217 # correct leader was elected
2218 correctCandidateResult = main.TRUE
2219 else:
2220 correctCandidateResult = main.FALSE
2221 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
2222 newLeader, oldLeaders[ 0 ][ 2 ] ) )
2223 else:
2224 main.log.warn( "Could not determine who should be the correct leader" )
2225 main.log.debug( oldLeaders[ 0 ] )
acsmars71adceb2015-08-31 15:09:26 -07002226 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002227 utilities.assert_equals(
2228 expect=main.TRUE,
2229 actual=correctCandidateResult,
2230 onpass="Correct Candidate Elected",
2231 onfail="Incorrect Candidate Elected" )
2232
Jon Hall85794ff2015-07-08 14:12:30 -07002233 main.step( "Run for election on old leader( just so everyone " +
2234 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002235 if oldLeaderCLI is not None:
2236 runResult = oldLeaderCLI.electionTestRun()
Jon Hall85794ff2015-07-08 14:12:30 -07002237 else:
acsmars71adceb2015-08-31 15:09:26 -07002238 main.log.error( "No old leader to re-elect" )
Jon Hall85794ff2015-07-08 14:12:30 -07002239 runResult = main.FALSE
2240 utilities.assert_equals(
2241 expect=main.TRUE,
2242 actual=runResult,
2243 onpass="App re-ran for election",
2244 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07002245
acsmars71adceb2015-08-31 15:09:26 -07002246 main.step(
2247 "Check that oldLeader is a candidate, and leader if only 1 node" )
2248 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07002249 # Get new leaders and candidates
2250 reRunLeaders = []
2251 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07002252 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07002253
2254 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07002255 if not reRunLeaders[0]:
2256 positionResult = main.FALSE
2257 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07002258 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
2259 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07002260 positionResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07002261 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002262 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002263 actual=positionResult,
Jon Hall85794ff2015-07-08 14:12:30 -07002264 onpass="Old leader successfully re-ran for election",
2265 onfail="Something went wrong with Leadership election after " +
2266 "the old leader re-ran for election" )
2267
2268 def CASE16( self, main ):
2269 """
2270 Install Distributed Primitives app
2271 """
Jon Halla440e872016-03-31 15:15:50 -07002272 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002273 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002274 assert main, "main not defined"
2275 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002276 assert main.CLIs, "main.CLIs not defined"
2277 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002278
2279 # Variables for the distributed primitives tests
2280 global pCounterName
Jon Hall85794ff2015-07-08 14:12:30 -07002281 global pCounterValue
Jon Hall85794ff2015-07-08 14:12:30 -07002282 global onosSet
2283 global onosSetName
2284 pCounterName = "TestON-Partitions"
Jon Hall85794ff2015-07-08 14:12:30 -07002285 pCounterValue = 0
Jon Hall85794ff2015-07-08 14:12:30 -07002286 onosSet = set([])
2287 onosSetName = "TestON-set"
2288
2289 description = "Install Primitives app"
2290 main.case( description )
2291 main.step( "Install Primitives app" )
2292 appName = "org.onosproject.distributedprimitives"
Jon Halla440e872016-03-31 15:15:50 -07002293 node = main.activeNodes[0]
2294 appResults = main.CLIs[node].activateApp( appName )
Jon Hall85794ff2015-07-08 14:12:30 -07002295 utilities.assert_equals( expect=main.TRUE,
2296 actual=appResults,
2297 onpass="Primitives app activated",
2298 onfail="Primitives app not activated" )
Jon Halla440e872016-03-31 15:15:50 -07002299 time.sleep( 5 ) # To allow all nodes to activate
Jon Hall85794ff2015-07-08 14:12:30 -07002300
2301 def CASE17( self, main ):
2302 """
2303 Check for basic functionality with distributed primitives
2304 """
Jon Hall85794ff2015-07-08 14:12:30 -07002305 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002306 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002307 assert main, "main not defined"
2308 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002309 assert main.CLIs, "main.CLIs not defined"
2310 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002311 assert pCounterName, "pCounterName not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002312 assert onosSetName, "onosSetName not defined"
2313 # NOTE: assert fails if value is 0/None/Empty/False
2314 try:
2315 pCounterValue
2316 except NameError:
2317 main.log.error( "pCounterValue not defined, setting to 0" )
2318 pCounterValue = 0
2319 try:
Jon Hall85794ff2015-07-08 14:12:30 -07002320 onosSet
2321 except NameError:
2322 main.log.error( "onosSet not defined, setting to empty Set" )
2323 onosSet = set([])
2324 # Variables for the distributed primitives tests. These are local only
2325 addValue = "a"
2326 addAllValue = "a b c d e f"
2327 retainValue = "c d e f"
2328
2329 description = "Check for basic functionality with distributed " +\
2330 "primitives"
2331 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07002332 main.caseExplanation = "Test the methods of the distributed " +\
2333 "primitives (counters and sets) throught the cli"
Jon Hall85794ff2015-07-08 14:12:30 -07002334 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07002335 # Partitioned counters
2336 main.step( "Increment then get a default counter on each node" )
Jon Hall85794ff2015-07-08 14:12:30 -07002337 pCounters = []
2338 threads = []
2339 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002340 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002341 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2342 name="counterAddAndGet-" + str( i ),
Jon Hall85794ff2015-07-08 14:12:30 -07002343 args=[ pCounterName ] )
2344 pCounterValue += 1
2345 addedPValues.append( pCounterValue )
2346 threads.append( t )
2347 t.start()
2348
2349 for t in threads:
2350 t.join()
2351 pCounters.append( t.result )
2352 # Check that counter incremented numController times
2353 pCounterResults = True
2354 for i in addedPValues:
2355 tmpResult = i in pCounters
2356 pCounterResults = pCounterResults and tmpResult
2357 if not tmpResult:
2358 main.log.error( str( i ) + " is not in partitioned "
2359 "counter incremented results" )
2360 utilities.assert_equals( expect=True,
2361 actual=pCounterResults,
2362 onpass="Default counter incremented",
2363 onfail="Error incrementing default" +
2364 " counter" )
2365
Jon Halle1a3b752015-07-22 13:02:46 -07002366 main.step( "Get then Increment a default counter on each node" )
2367 pCounters = []
2368 threads = []
2369 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002370 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002371 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2372 name="counterGetAndAdd-" + str( i ),
2373 args=[ pCounterName ] )
2374 addedPValues.append( pCounterValue )
2375 pCounterValue += 1
2376 threads.append( t )
2377 t.start()
2378
2379 for t in threads:
2380 t.join()
2381 pCounters.append( t.result )
2382 # Check that counter incremented numController times
2383 pCounterResults = True
2384 for i in addedPValues:
2385 tmpResult = i in pCounters
2386 pCounterResults = pCounterResults and tmpResult
2387 if not tmpResult:
2388 main.log.error( str( i ) + " is not in partitioned "
2389 "counter incremented results" )
2390 utilities.assert_equals( expect=True,
2391 actual=pCounterResults,
2392 onpass="Default counter incremented",
2393 onfail="Error incrementing default" +
2394 " counter" )
2395
2396 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07002397 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07002398 utilities.assert_equals( expect=main.TRUE,
2399 actual=incrementCheck,
2400 onpass="Added counters are correct",
2401 onfail="Added counters are incorrect" )
2402
2403 main.step( "Add -8 to then get a default counter on each node" )
2404 pCounters = []
2405 threads = []
2406 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002407 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002408 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2409 name="counterIncrement-" + str( i ),
2410 args=[ pCounterName ],
2411 kwargs={ "delta": -8 } )
2412 pCounterValue += -8
2413 addedPValues.append( pCounterValue )
2414 threads.append( t )
2415 t.start()
2416
2417 for t in threads:
2418 t.join()
2419 pCounters.append( t.result )
2420 # Check that counter incremented numController times
2421 pCounterResults = True
2422 for i in addedPValues:
2423 tmpResult = i in pCounters
2424 pCounterResults = pCounterResults and tmpResult
2425 if not tmpResult:
2426 main.log.error( str( i ) + " is not in partitioned "
2427 "counter incremented results" )
2428 utilities.assert_equals( expect=True,
2429 actual=pCounterResults,
2430 onpass="Default counter incremented",
2431 onfail="Error incrementing default" +
2432 " counter" )
2433
2434 main.step( "Add 5 to then get a default counter on each node" )
2435 pCounters = []
2436 threads = []
2437 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002438 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002439 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2440 name="counterIncrement-" + str( i ),
2441 args=[ pCounterName ],
2442 kwargs={ "delta": 5 } )
2443 pCounterValue += 5
2444 addedPValues.append( pCounterValue )
2445 threads.append( t )
2446 t.start()
2447
2448 for t in threads:
2449 t.join()
2450 pCounters.append( t.result )
2451 # Check that counter incremented numController times
2452 pCounterResults = True
2453 for i in addedPValues:
2454 tmpResult = i in pCounters
2455 pCounterResults = pCounterResults and tmpResult
2456 if not tmpResult:
2457 main.log.error( str( i ) + " is not in partitioned "
2458 "counter incremented results" )
2459 utilities.assert_equals( expect=True,
2460 actual=pCounterResults,
2461 onpass="Default counter incremented",
2462 onfail="Error incrementing default" +
2463 " counter" )
2464
2465 main.step( "Get then add 5 to a default counter on each node" )
2466 pCounters = []
2467 threads = []
2468 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002469 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002470 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2471 name="counterIncrement-" + str( i ),
2472 args=[ pCounterName ],
2473 kwargs={ "delta": 5 } )
2474 addedPValues.append( pCounterValue )
2475 pCounterValue += 5
2476 threads.append( t )
2477 t.start()
2478
2479 for t in threads:
2480 t.join()
2481 pCounters.append( t.result )
2482 # Check that counter incremented numController times
2483 pCounterResults = True
2484 for i in addedPValues:
2485 tmpResult = i in pCounters
2486 pCounterResults = pCounterResults and tmpResult
2487 if not tmpResult:
2488 main.log.error( str( i ) + " is not in partitioned "
2489 "counter incremented results" )
2490 utilities.assert_equals( expect=True,
2491 actual=pCounterResults,
2492 onpass="Default counter incremented",
2493 onfail="Error incrementing default" +
2494 " counter" )
2495
2496 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07002497 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07002498 utilities.assert_equals( expect=main.TRUE,
2499 actual=incrementCheck,
2500 onpass="Added counters are correct",
2501 onfail="Added counters are incorrect" )
2502
Jon Hall85794ff2015-07-08 14:12:30 -07002503 # DISTRIBUTED SETS
2504 main.step( "Distributed Set get" )
2505 size = len( onosSet )
2506 getResponses = []
2507 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002508 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002509 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002510 name="setTestGet-" + str( i ),
2511 args=[ onosSetName ] )
2512 threads.append( t )
2513 t.start()
2514 for t in threads:
2515 t.join()
2516 getResponses.append( t.result )
2517
2518 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002519 for i in range( len( main.activeNodes ) ):
2520 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002521 if isinstance( getResponses[ i ], list):
2522 current = set( getResponses[ i ] )
2523 if len( current ) == len( getResponses[ i ] ):
2524 # no repeats
2525 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002526 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002527 " has incorrect view" +
2528 " of set " + onosSetName + ":\n" +
2529 str( getResponses[ i ] ) )
2530 main.log.debug( "Expected: " + str( onosSet ) )
2531 main.log.debug( "Actual: " + str( current ) )
2532 getResults = main.FALSE
2533 else:
2534 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002535 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002536 " has repeat elements in" +
2537 " set " + onosSetName + ":\n" +
2538 str( getResponses[ i ] ) )
2539 getResults = main.FALSE
2540 elif getResponses[ i ] == main.ERROR:
2541 getResults = main.FALSE
2542 utilities.assert_equals( expect=main.TRUE,
2543 actual=getResults,
2544 onpass="Set elements are correct",
2545 onfail="Set elements are incorrect" )
2546
2547 main.step( "Distributed Set size" )
2548 sizeResponses = []
2549 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002550 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002551 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002552 name="setTestSize-" + str( i ),
2553 args=[ onosSetName ] )
2554 threads.append( t )
2555 t.start()
2556 for t in threads:
2557 t.join()
2558 sizeResponses.append( t.result )
2559
2560 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002561 for i in range( len( main.activeNodes ) ):
2562 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002563 if size != sizeResponses[ i ]:
2564 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002565 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002566 " expected a size of " + str( size ) +
2567 " for set " + onosSetName +
2568 " but got " + str( sizeResponses[ i ] ) )
2569 utilities.assert_equals( expect=main.TRUE,
2570 actual=sizeResults,
2571 onpass="Set sizes are correct",
2572 onfail="Set sizes are incorrect" )
2573
2574 main.step( "Distributed Set add()" )
2575 onosSet.add( addValue )
2576 addResponses = []
2577 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002578 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002579 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002580 name="setTestAdd-" + str( i ),
2581 args=[ onosSetName, addValue ] )
2582 threads.append( t )
2583 t.start()
2584 for t in threads:
2585 t.join()
2586 addResponses.append( t.result )
2587
2588 # main.TRUE = successfully changed the set
2589 # main.FALSE = action resulted in no change in set
2590 # main.ERROR - Some error in executing the function
2591 addResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002592 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002593 if addResponses[ i ] == main.TRUE:
2594 # All is well
2595 pass
2596 elif addResponses[ i ] == main.FALSE:
2597 # Already in set, probably fine
2598 pass
2599 elif addResponses[ i ] == main.ERROR:
2600 # Error in execution
2601 addResults = main.FALSE
2602 else:
2603 # unexpected result
2604 addResults = main.FALSE
2605 if addResults != main.TRUE:
2606 main.log.error( "Error executing set add" )
2607
2608 # Check if set is still correct
2609 size = len( onosSet )
2610 getResponses = []
2611 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002612 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002613 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002614 name="setTestGet-" + str( i ),
2615 args=[ onosSetName ] )
2616 threads.append( t )
2617 t.start()
2618 for t in threads:
2619 t.join()
2620 getResponses.append( t.result )
2621 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002622 for i in range( len( main.activeNodes ) ):
2623 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002624 if isinstance( getResponses[ i ], list):
2625 current = set( getResponses[ i ] )
2626 if len( current ) == len( getResponses[ i ] ):
2627 # no repeats
2628 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002629 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall85794ff2015-07-08 14:12:30 -07002630 " of set " + onosSetName + ":\n" +
2631 str( getResponses[ i ] ) )
2632 main.log.debug( "Expected: " + str( onosSet ) )
2633 main.log.debug( "Actual: " + str( current ) )
2634 getResults = main.FALSE
2635 else:
2636 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002637 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall85794ff2015-07-08 14:12:30 -07002638 " set " + onosSetName + ":\n" +
2639 str( getResponses[ i ] ) )
2640 getResults = main.FALSE
2641 elif getResponses[ i ] == main.ERROR:
2642 getResults = main.FALSE
2643 sizeResponses = []
2644 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002645 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002646 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002647 name="setTestSize-" + str( i ),
2648 args=[ onosSetName ] )
2649 threads.append( t )
2650 t.start()
2651 for t in threads:
2652 t.join()
2653 sizeResponses.append( t.result )
2654 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002655 for i in range( len( main.activeNodes ) ):
2656 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002657 if size != sizeResponses[ i ]:
2658 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002659 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002660 " expected a size of " + str( size ) +
2661 " for set " + onosSetName +
2662 " but got " + str( sizeResponses[ i ] ) )
2663 addResults = addResults and getResults and sizeResults
2664 utilities.assert_equals( expect=main.TRUE,
2665 actual=addResults,
2666 onpass="Set add correct",
2667 onfail="Set add was incorrect" )
2668
2669 main.step( "Distributed Set addAll()" )
2670 onosSet.update( addAllValue.split() )
2671 addResponses = []
2672 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002673 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002674 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002675 name="setTestAddAll-" + str( i ),
2676 args=[ onosSetName, addAllValue ] )
2677 threads.append( t )
2678 t.start()
2679 for t in threads:
2680 t.join()
2681 addResponses.append( t.result )
2682
2683 # main.TRUE = successfully changed the set
2684 # main.FALSE = action resulted in no change in set
2685 # main.ERROR - Some error in executing the function
2686 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002687 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002688 if addResponses[ i ] == main.TRUE:
2689 # All is well
2690 pass
2691 elif addResponses[ i ] == main.FALSE:
2692 # Already in set, probably fine
2693 pass
2694 elif addResponses[ i ] == main.ERROR:
2695 # Error in execution
2696 addAllResults = main.FALSE
2697 else:
2698 # unexpected result
2699 addAllResults = main.FALSE
2700 if addAllResults != main.TRUE:
2701 main.log.error( "Error executing set addAll" )
2702
2703 # Check if set is still correct
2704 size = len( onosSet )
2705 getResponses = []
2706 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002707 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002708 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002709 name="setTestGet-" + str( i ),
2710 args=[ onosSetName ] )
2711 threads.append( t )
2712 t.start()
2713 for t in threads:
2714 t.join()
2715 getResponses.append( t.result )
2716 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002717 for i in range( len( main.activeNodes ) ):
2718 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002719 if isinstance( getResponses[ i ], list):
2720 current = set( getResponses[ i ] )
2721 if len( current ) == len( getResponses[ i ] ):
2722 # no repeats
2723 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002724 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002725 " has incorrect view" +
2726 " of set " + onosSetName + ":\n" +
2727 str( getResponses[ i ] ) )
2728 main.log.debug( "Expected: " + str( onosSet ) )
2729 main.log.debug( "Actual: " + str( current ) )
2730 getResults = main.FALSE
2731 else:
2732 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002733 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002734 " has repeat elements in" +
2735 " set " + onosSetName + ":\n" +
2736 str( getResponses[ i ] ) )
2737 getResults = main.FALSE
2738 elif getResponses[ i ] == main.ERROR:
2739 getResults = main.FALSE
2740 sizeResponses = []
2741 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002742 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002743 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002744 name="setTestSize-" + str( i ),
2745 args=[ onosSetName ] )
2746 threads.append( t )
2747 t.start()
2748 for t in threads:
2749 t.join()
2750 sizeResponses.append( t.result )
2751 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002752 for i in range( len( main.activeNodes ) ):
2753 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002754 if size != sizeResponses[ i ]:
2755 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002756 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002757 " expected a size of " + str( size ) +
2758 " for set " + onosSetName +
2759 " but got " + str( sizeResponses[ i ] ) )
2760 addAllResults = addAllResults and getResults and sizeResults
2761 utilities.assert_equals( expect=main.TRUE,
2762 actual=addAllResults,
2763 onpass="Set addAll correct",
2764 onfail="Set addAll was incorrect" )
2765
2766 main.step( "Distributed Set contains()" )
2767 containsResponses = []
2768 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002769 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002770 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002771 name="setContains-" + str( i ),
2772 args=[ onosSetName ],
2773 kwargs={ "values": addValue } )
2774 threads.append( t )
2775 t.start()
2776 for t in threads:
2777 t.join()
2778 # NOTE: This is the tuple
2779 containsResponses.append( t.result )
2780
2781 containsResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002782 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002783 if containsResponses[ i ] == main.ERROR:
2784 containsResults = main.FALSE
2785 else:
2786 containsResults = containsResults and\
2787 containsResponses[ i ][ 1 ]
2788 utilities.assert_equals( expect=main.TRUE,
2789 actual=containsResults,
2790 onpass="Set contains is functional",
2791 onfail="Set contains failed" )
2792
2793 main.step( "Distributed Set containsAll()" )
2794 containsAllResponses = []
2795 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002796 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002797 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002798 name="setContainsAll-" + str( i ),
2799 args=[ onosSetName ],
2800 kwargs={ "values": addAllValue } )
2801 threads.append( t )
2802 t.start()
2803 for t in threads:
2804 t.join()
2805 # NOTE: This is the tuple
2806 containsAllResponses.append( t.result )
2807
2808 containsAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002809 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002810 if containsResponses[ i ] == main.ERROR:
2811 containsResults = main.FALSE
2812 else:
2813 containsResults = containsResults and\
2814 containsResponses[ i ][ 1 ]
2815 utilities.assert_equals( expect=main.TRUE,
2816 actual=containsAllResults,
2817 onpass="Set containsAll is functional",
2818 onfail="Set containsAll failed" )
2819
2820 main.step( "Distributed Set remove()" )
2821 onosSet.remove( addValue )
2822 removeResponses = []
2823 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002824 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002825 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002826 name="setTestRemove-" + str( i ),
2827 args=[ onosSetName, addValue ] )
2828 threads.append( t )
2829 t.start()
2830 for t in threads:
2831 t.join()
2832 removeResponses.append( t.result )
2833
2834 # main.TRUE = successfully changed the set
2835 # main.FALSE = action resulted in no change in set
2836 # main.ERROR - Some error in executing the function
2837 removeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002838 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002839 if removeResponses[ i ] == main.TRUE:
2840 # All is well
2841 pass
2842 elif removeResponses[ i ] == main.FALSE:
2843 # not in set, probably fine
2844 pass
2845 elif removeResponses[ i ] == main.ERROR:
2846 # Error in execution
2847 removeResults = main.FALSE
2848 else:
2849 # unexpected result
2850 removeResults = main.FALSE
2851 if removeResults != main.TRUE:
2852 main.log.error( "Error executing set remove" )
2853
2854 # Check if set is still correct
2855 size = len( onosSet )
2856 getResponses = []
2857 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002858 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002859 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002860 name="setTestGet-" + str( i ),
2861 args=[ onosSetName ] )
2862 threads.append( t )
2863 t.start()
2864 for t in threads:
2865 t.join()
2866 getResponses.append( t.result )
2867 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002868 for i in range( len( main.activeNodes ) ):
2869 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002870 if isinstance( getResponses[ i ], list):
2871 current = set( getResponses[ i ] )
2872 if len( current ) == len( getResponses[ i ] ):
2873 # no repeats
2874 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002875 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002876 " has incorrect view" +
2877 " of set " + onosSetName + ":\n" +
2878 str( getResponses[ i ] ) )
2879 main.log.debug( "Expected: " + str( onosSet ) )
2880 main.log.debug( "Actual: " + str( current ) )
2881 getResults = main.FALSE
2882 else:
2883 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002884 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002885 " has repeat elements in" +
2886 " set " + onosSetName + ":\n" +
2887 str( getResponses[ i ] ) )
2888 getResults = main.FALSE
2889 elif getResponses[ i ] == main.ERROR:
2890 getResults = main.FALSE
2891 sizeResponses = []
2892 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002893 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002894 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002895 name="setTestSize-" + str( i ),
2896 args=[ onosSetName ] )
2897 threads.append( t )
2898 t.start()
2899 for t in threads:
2900 t.join()
2901 sizeResponses.append( t.result )
2902 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002903 for i in range( len( main.activeNodes ) ):
2904 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002905 if size != sizeResponses[ i ]:
2906 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002907 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002908 " expected a size of " + str( size ) +
2909 " for set " + onosSetName +
2910 " but got " + str( sizeResponses[ i ] ) )
2911 removeResults = removeResults and getResults and sizeResults
2912 utilities.assert_equals( expect=main.TRUE,
2913 actual=removeResults,
2914 onpass="Set remove correct",
2915 onfail="Set remove was incorrect" )
2916
2917 main.step( "Distributed Set removeAll()" )
2918 onosSet.difference_update( addAllValue.split() )
2919 removeAllResponses = []
2920 threads = []
2921 try:
Jon Halla440e872016-03-31 15:15:50 -07002922 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002923 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002924 name="setTestRemoveAll-" + str( i ),
2925 args=[ onosSetName, addAllValue ] )
2926 threads.append( t )
2927 t.start()
2928 for t in threads:
2929 t.join()
2930 removeAllResponses.append( t.result )
2931 except Exception, e:
2932 main.log.exception(e)
2933
2934 # main.TRUE = successfully changed the set
2935 # main.FALSE = action resulted in no change in set
2936 # main.ERROR - Some error in executing the function
2937 removeAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002938 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002939 if removeAllResponses[ i ] == main.TRUE:
2940 # All is well
2941 pass
2942 elif removeAllResponses[ i ] == main.FALSE:
2943 # not in set, probably fine
2944 pass
2945 elif removeAllResponses[ i ] == main.ERROR:
2946 # Error in execution
2947 removeAllResults = main.FALSE
2948 else:
2949 # unexpected result
2950 removeAllResults = main.FALSE
2951 if removeAllResults != main.TRUE:
2952 main.log.error( "Error executing set removeAll" )
2953
2954 # Check if set is still correct
2955 size = len( onosSet )
2956 getResponses = []
2957 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002958 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002959 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002960 name="setTestGet-" + str( i ),
2961 args=[ onosSetName ] )
2962 threads.append( t )
2963 t.start()
2964 for t in threads:
2965 t.join()
2966 getResponses.append( t.result )
2967 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002968 for i in range( len( main.activeNodes ) ):
2969 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002970 if isinstance( getResponses[ i ], list):
2971 current = set( getResponses[ i ] )
2972 if len( current ) == len( getResponses[ i ] ):
2973 # no repeats
2974 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002975 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002976 " has incorrect view" +
2977 " of set " + onosSetName + ":\n" +
2978 str( getResponses[ i ] ) )
2979 main.log.debug( "Expected: " + str( onosSet ) )
2980 main.log.debug( "Actual: " + str( current ) )
2981 getResults = main.FALSE
2982 else:
2983 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002984 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002985 " has repeat elements in" +
2986 " set " + onosSetName + ":\n" +
2987 str( getResponses[ i ] ) )
2988 getResults = main.FALSE
2989 elif getResponses[ i ] == main.ERROR:
2990 getResults = main.FALSE
2991 sizeResponses = []
2992 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002993 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002994 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002995 name="setTestSize-" + str( i ),
2996 args=[ onosSetName ] )
2997 threads.append( t )
2998 t.start()
2999 for t in threads:
3000 t.join()
3001 sizeResponses.append( t.result )
3002 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003003 for i in range( len( main.activeNodes ) ):
3004 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003005 if size != sizeResponses[ i ]:
3006 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003007 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003008 " expected a size of " + str( size ) +
3009 " for set " + onosSetName +
3010 " but got " + str( sizeResponses[ i ] ) )
3011 removeAllResults = removeAllResults and getResults and sizeResults
3012 utilities.assert_equals( expect=main.TRUE,
3013 actual=removeAllResults,
3014 onpass="Set removeAll correct",
3015 onfail="Set removeAll was incorrect" )
3016
3017 main.step( "Distributed Set addAll()" )
3018 onosSet.update( addAllValue.split() )
3019 addResponses = []
3020 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003021 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003022 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003023 name="setTestAddAll-" + str( i ),
3024 args=[ onosSetName, addAllValue ] )
3025 threads.append( t )
3026 t.start()
3027 for t in threads:
3028 t.join()
3029 addResponses.append( t.result )
3030
3031 # main.TRUE = successfully changed the set
3032 # main.FALSE = action resulted in no change in set
3033 # main.ERROR - Some error in executing the function
3034 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003035 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07003036 if addResponses[ i ] == main.TRUE:
3037 # All is well
3038 pass
3039 elif addResponses[ i ] == main.FALSE:
3040 # Already in set, probably fine
3041 pass
3042 elif addResponses[ i ] == main.ERROR:
3043 # Error in execution
3044 addAllResults = main.FALSE
3045 else:
3046 # unexpected result
3047 addAllResults = main.FALSE
3048 if addAllResults != main.TRUE:
3049 main.log.error( "Error executing set addAll" )
3050
3051 # Check if set is still correct
3052 size = len( onosSet )
3053 getResponses = []
3054 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003055 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003056 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003057 name="setTestGet-" + str( i ),
3058 args=[ onosSetName ] )
3059 threads.append( t )
3060 t.start()
3061 for t in threads:
3062 t.join()
3063 getResponses.append( t.result )
3064 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003065 for i in range( len( main.activeNodes ) ):
3066 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003067 if isinstance( getResponses[ i ], list):
3068 current = set( getResponses[ i ] )
3069 if len( current ) == len( getResponses[ i ] ):
3070 # no repeats
3071 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07003072 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003073 " has incorrect view" +
3074 " of set " + onosSetName + ":\n" +
3075 str( getResponses[ i ] ) )
3076 main.log.debug( "Expected: " + str( onosSet ) )
3077 main.log.debug( "Actual: " + str( current ) )
3078 getResults = main.FALSE
3079 else:
3080 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07003081 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003082 " has repeat elements in" +
3083 " set " + onosSetName + ":\n" +
3084 str( getResponses[ i ] ) )
3085 getResults = main.FALSE
3086 elif getResponses[ i ] == main.ERROR:
3087 getResults = main.FALSE
3088 sizeResponses = []
3089 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003090 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003091 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003092 name="setTestSize-" + str( i ),
3093 args=[ onosSetName ] )
3094 threads.append( t )
3095 t.start()
3096 for t in threads:
3097 t.join()
3098 sizeResponses.append( t.result )
3099 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003100 for i in range( len( main.activeNodes ) ):
3101 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003102 if size != sizeResponses[ i ]:
3103 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003104 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003105 " expected a size of " + str( size ) +
3106 " for set " + onosSetName +
3107 " but got " + str( sizeResponses[ i ] ) )
3108 addAllResults = addAllResults and getResults and sizeResults
3109 utilities.assert_equals( expect=main.TRUE,
3110 actual=addAllResults,
3111 onpass="Set addAll correct",
3112 onfail="Set addAll was incorrect" )
3113
3114 main.step( "Distributed Set clear()" )
3115 onosSet.clear()
3116 clearResponses = []
3117 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003118 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003119 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003120 name="setTestClear-" + str( i ),
3121 args=[ onosSetName, " "], # Values doesn't matter
3122 kwargs={ "clear": True } )
3123 threads.append( t )
3124 t.start()
3125 for t in threads:
3126 t.join()
3127 clearResponses.append( t.result )
3128
3129 # main.TRUE = successfully changed the set
3130 # main.FALSE = action resulted in no change in set
3131 # main.ERROR - Some error in executing the function
3132 clearResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003133 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07003134 if clearResponses[ i ] == main.TRUE:
3135 # All is well
3136 pass
3137 elif clearResponses[ i ] == main.FALSE:
3138 # Nothing set, probably fine
3139 pass
3140 elif clearResponses[ i ] == main.ERROR:
3141 # Error in execution
3142 clearResults = main.FALSE
3143 else:
3144 # unexpected result
3145 clearResults = main.FALSE
3146 if clearResults != main.TRUE:
3147 main.log.error( "Error executing set clear" )
3148
3149 # Check if set is still correct
3150 size = len( onosSet )
3151 getResponses = []
3152 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003153 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003154 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003155 name="setTestGet-" + str( i ),
3156 args=[ onosSetName ] )
3157 threads.append( t )
3158 t.start()
3159 for t in threads:
3160 t.join()
3161 getResponses.append( t.result )
3162 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003163 for i in range( len( main.activeNodes ) ):
3164 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003165 if isinstance( getResponses[ i ], list):
3166 current = set( getResponses[ i ] )
3167 if len( current ) == len( getResponses[ i ] ):
3168 # no repeats
3169 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07003170 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003171 " has incorrect view" +
3172 " of set " + onosSetName + ":\n" +
3173 str( getResponses[ i ] ) )
3174 main.log.debug( "Expected: " + str( onosSet ) )
3175 main.log.debug( "Actual: " + str( current ) )
3176 getResults = main.FALSE
3177 else:
3178 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07003179 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003180 " has repeat elements in" +
3181 " set " + onosSetName + ":\n" +
3182 str( getResponses[ i ] ) )
3183 getResults = main.FALSE
3184 elif getResponses[ i ] == main.ERROR:
3185 getResults = main.FALSE
3186 sizeResponses = []
3187 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003188 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003189 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003190 name="setTestSize-" + str( i ),
3191 args=[ onosSetName ] )
3192 threads.append( t )
3193 t.start()
3194 for t in threads:
3195 t.join()
3196 sizeResponses.append( t.result )
3197 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003198 for i in range( len( main.activeNodes ) ):
3199 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003200 if size != sizeResponses[ i ]:
3201 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003202 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003203 " expected a size of " + str( size ) +
3204 " for set " + onosSetName +
3205 " but got " + str( sizeResponses[ i ] ) )
3206 clearResults = clearResults and getResults and sizeResults
3207 utilities.assert_equals( expect=main.TRUE,
3208 actual=clearResults,
3209 onpass="Set clear correct",
3210 onfail="Set clear was incorrect" )
3211
3212 main.step( "Distributed Set addAll()" )
3213 onosSet.update( addAllValue.split() )
3214 addResponses = []
3215 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003216 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003217 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003218 name="setTestAddAll-" + str( i ),
3219 args=[ onosSetName, addAllValue ] )
3220 threads.append( t )
3221 t.start()
3222 for t in threads:
3223 t.join()
3224 addResponses.append( t.result )
3225
3226 # main.TRUE = successfully changed the set
3227 # main.FALSE = action resulted in no change in set
3228 # main.ERROR - Some error in executing the function
3229 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003230 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07003231 if addResponses[ i ] == main.TRUE:
3232 # All is well
3233 pass
3234 elif addResponses[ i ] == main.FALSE:
3235 # Already in set, probably fine
3236 pass
3237 elif addResponses[ i ] == main.ERROR:
3238 # Error in execution
3239 addAllResults = main.FALSE
3240 else:
3241 # unexpected result
3242 addAllResults = main.FALSE
3243 if addAllResults != main.TRUE:
3244 main.log.error( "Error executing set addAll" )
3245
3246 # Check if set is still correct
3247 size = len( onosSet )
3248 getResponses = []
3249 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003250 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003251 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003252 name="setTestGet-" + str( i ),
3253 args=[ onosSetName ] )
3254 threads.append( t )
3255 t.start()
3256 for t in threads:
3257 t.join()
3258 getResponses.append( t.result )
3259 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003260 for i in range( len( main.activeNodes ) ):
3261 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003262 if isinstance( getResponses[ i ], list):
3263 current = set( getResponses[ i ] )
3264 if len( current ) == len( getResponses[ i ] ):
3265 # no repeats
3266 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07003267 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003268 " has incorrect view" +
3269 " of set " + onosSetName + ":\n" +
3270 str( getResponses[ i ] ) )
3271 main.log.debug( "Expected: " + str( onosSet ) )
3272 main.log.debug( "Actual: " + str( current ) )
3273 getResults = main.FALSE
3274 else:
3275 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07003276 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003277 " has repeat elements in" +
3278 " set " + onosSetName + ":\n" +
3279 str( getResponses[ i ] ) )
3280 getResults = main.FALSE
3281 elif getResponses[ i ] == main.ERROR:
3282 getResults = main.FALSE
3283 sizeResponses = []
3284 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003285 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003286 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003287 name="setTestSize-" + str( i ),
3288 args=[ onosSetName ] )
3289 threads.append( t )
3290 t.start()
3291 for t in threads:
3292 t.join()
3293 sizeResponses.append( t.result )
3294 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003295 for i in range( len( main.activeNodes ) ):
3296 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003297 if size != sizeResponses[ i ]:
3298 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003299 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003300 " expected a size of " + str( size ) +
3301 " for set " + onosSetName +
3302 " but got " + str( sizeResponses[ i ] ) )
3303 addAllResults = addAllResults and getResults and sizeResults
3304 utilities.assert_equals( expect=main.TRUE,
3305 actual=addAllResults,
3306 onpass="Set addAll correct",
3307 onfail="Set addAll was incorrect" )
3308
3309 main.step( "Distributed Set retain()" )
3310 onosSet.intersection_update( retainValue.split() )
3311 retainResponses = []
3312 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003313 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003314 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003315 name="setTestRetain-" + str( i ),
3316 args=[ onosSetName, retainValue ],
3317 kwargs={ "retain": True } )
3318 threads.append( t )
3319 t.start()
3320 for t in threads:
3321 t.join()
3322 retainResponses.append( t.result )
3323
3324 # main.TRUE = successfully changed the set
3325 # main.FALSE = action resulted in no change in set
3326 # main.ERROR - Some error in executing the function
3327 retainResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003328 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07003329 if retainResponses[ i ] == main.TRUE:
3330 # All is well
3331 pass
3332 elif retainResponses[ i ] == main.FALSE:
3333 # Already in set, probably fine
3334 pass
3335 elif retainResponses[ i ] == main.ERROR:
3336 # Error in execution
3337 retainResults = main.FALSE
3338 else:
3339 # unexpected result
3340 retainResults = main.FALSE
3341 if retainResults != main.TRUE:
3342 main.log.error( "Error executing set retain" )
3343
3344 # Check if set is still correct
3345 size = len( onosSet )
3346 getResponses = []
3347 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003348 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003349 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003350 name="setTestGet-" + str( i ),
3351 args=[ onosSetName ] )
3352 threads.append( t )
3353 t.start()
3354 for t in threads:
3355 t.join()
3356 getResponses.append( t.result )
3357 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003358 for i in range( len( main.activeNodes ) ):
3359 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003360 if isinstance( getResponses[ i ], list):
3361 current = set( getResponses[ i ] )
3362 if len( current ) == len( getResponses[ i ] ):
3363 # no repeats
3364 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07003365 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003366 " has incorrect view" +
3367 " of set " + onosSetName + ":\n" +
3368 str( getResponses[ i ] ) )
3369 main.log.debug( "Expected: " + str( onosSet ) )
3370 main.log.debug( "Actual: " + str( current ) )
3371 getResults = main.FALSE
3372 else:
3373 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07003374 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003375 " has repeat elements in" +
3376 " set " + onosSetName + ":\n" +
3377 str( getResponses[ i ] ) )
3378 getResults = main.FALSE
3379 elif getResponses[ i ] == main.ERROR:
3380 getResults = main.FALSE
3381 sizeResponses = []
3382 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003383 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003384 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003385 name="setTestSize-" + str( i ),
3386 args=[ onosSetName ] )
3387 threads.append( t )
3388 t.start()
3389 for t in threads:
3390 t.join()
3391 sizeResponses.append( t.result )
3392 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003393 for i in range( len( main.activeNodes ) ):
3394 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003395 if size != sizeResponses[ i ]:
3396 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003397 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall85794ff2015-07-08 14:12:30 -07003398 str( size ) + " for set " + onosSetName +
3399 " but got " + str( sizeResponses[ i ] ) )
3400 retainResults = retainResults and getResults and sizeResults
3401 utilities.assert_equals( expect=main.TRUE,
3402 actual=retainResults,
3403 onpass="Set retain correct",
3404 onfail="Set retain was incorrect" )
3405
Jon Hall2a5002c2015-08-21 16:49:11 -07003406 # Transactional maps
3407 main.step( "Partitioned Transactional maps put" )
3408 tMapValue = "Testing"
3409 numKeys = 100
3410 putResult = True
Jon Halla440e872016-03-31 15:15:50 -07003411 node = main.activeNodes[0]
3412 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
3413 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07003414 for i in putResponses:
3415 if putResponses[ i ][ 'value' ] != tMapValue:
3416 putResult = False
3417 else:
3418 putResult = False
3419 if not putResult:
3420 main.log.debug( "Put response values: " + str( putResponses ) )
3421 utilities.assert_equals( expect=True,
3422 actual=putResult,
3423 onpass="Partitioned Transactional Map put successful",
3424 onfail="Partitioned Transactional Map put values are incorrect" )
3425
3426 main.step( "Partitioned Transactional maps get" )
Jon Hall9bfadd22016-05-11 14:48:07 -07003427 # FIXME: is this sleep needed?
3428 time.sleep( 5 )
3429
Jon Hall2a5002c2015-08-21 16:49:11 -07003430 getCheck = True
3431 for n in range( 1, numKeys + 1 ):
3432 getResponses = []
3433 threads = []
3434 valueCheck = True
Jon Halla440e872016-03-31 15:15:50 -07003435 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07003436 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
3437 name="TMap-get-" + str( i ),
Jon Halla440e872016-03-31 15:15:50 -07003438 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07003439 threads.append( t )
3440 t.start()
3441 for t in threads:
3442 t.join()
3443 getResponses.append( t.result )
3444 for node in getResponses:
3445 if node != tMapValue:
3446 valueCheck = False
3447 if not valueCheck:
3448 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
3449 main.log.warn( getResponses )
3450 getCheck = getCheck and valueCheck
3451 utilities.assert_equals( expect=True,
3452 actual=getCheck,
3453 onpass="Partitioned Transactional Map get values were correct",
3454 onfail="Partitioned Transactional Map values incorrect" )