blob: bd581ae500dab49b0a2df1b179308226c5981bcb [file] [log] [blame]
Jon Hall85794ff2015-07-08 14:12:30 -07001"""
2Description: This test is to determine if a single
3 instance ONOS 'cluster' can handle a restart
4
5List of test cases:
6CASE1: Compile ONOS and push it to the test machines
7CASE2: Assign devices to controllers
8CASE21: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case.
13CASE7: Check state after control plane failure
14CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
20CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
22CASE16: Install Distributed Primitives app
23CASE17: Check for basic functionality with distributed primitives
24"""
25
26
27class HAsingleInstanceRestart:
28
29 def __init__( self ):
30 self.default = ''
31
32 def CASE1( self, main ):
33 """
34 CASE1 is to compile ONOS and push it to the test machines
35
36 Startup sequence:
37 cell <name>
38 onos-verify-cell
39 NOTE: temporary - onos-remove-raft-logs
40 onos-uninstall
41 start mininet
42 git pull
43 mvn clean install
44 onos-package
45 onos-install -f
46 onos-wait-for-start
47 start cli sessions
48 start tcpdump
49 """
Jon Halle1a3b752015-07-22 13:02:46 -070050 import imp
Jon Hallf3d16e72015-12-16 17:45:08 -080051 import time
Jon Halla440e872016-03-31 15:15:50 -070052 import json
Jon Hall85794ff2015-07-08 14:12:30 -070053 main.log.info( "ONOS Single node cluster restart " +
54 "HA test - initialization" )
55 main.case( "Setting up test environment" )
Jon Hall783bbf92015-07-23 14:33:19 -070056 main.caseExplanation = "Setup the test environment including " +\
Jon Hall85794ff2015-07-08 14:12:30 -070057 "installing ONOS, starting Mininet and ONOS" +\
58 "cli sessions."
Jon Hall85794ff2015-07-08 14:12:30 -070059
60 # load some variables from the params file
61 PULLCODE = False
62 if main.params[ 'Git' ] == 'True':
63 PULLCODE = True
64 gitBranch = main.params[ 'branch' ]
65 cellName = main.params[ 'ENV' ][ 'cellName' ]
66
Jon Halle1a3b752015-07-22 13:02:46 -070067 main.numCtrls = int( main.params[ 'num_controllers' ] )
Jon Hall5cf14d52015-07-16 12:15:19 -070068 if main.ONOSbench.maxNodes:
Jon Halle1a3b752015-07-22 13:02:46 -070069 if main.ONOSbench.maxNodes < main.numCtrls:
70 main.numCtrls = int( main.ONOSbench.maxNodes )
Jon Hall85794ff2015-07-08 14:12:30 -070071
Jon Halle1a3b752015-07-22 13:02:46 -070072 try:
Jon Hall53c5e662016-04-13 16:06:56 -070073 from tests.HA.dependencies.HA import HA
Jon Hall41d39f12016-04-11 22:54:35 -070074 main.HA = HA()
Jon Halle1a3b752015-07-22 13:02:46 -070075 except Exception as e:
76 main.log.exception( e )
77 main.cleanup()
78 main.exit()
79
80 main.CLIs = []
81 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -070082 ipList = []
83 for i in range( 1, int( main.ONOSbench.maxNodes ) + 1 ):
84 try:
Jon Halle1a3b752015-07-22 13:02:46 -070085 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
86 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
87 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -070088 except AttributeError:
89 break
Jon Hall85794ff2015-07-08 14:12:30 -070090
Jon Hall5cf14d52015-07-16 12:15:19 -070091 main.step( "Create cell file" )
92 cellAppString = main.params[ 'ENV' ][ 'appString' ]
93 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
94 main.Mininet1.ip_address,
95 cellAppString, ipList )
Jon Hall85794ff2015-07-08 14:12:30 -070096 main.step( "Applying cell variable to environment" )
97 cellResult = main.ONOSbench.setCell( cellName )
98 verifyResult = main.ONOSbench.verifyCell()
99
100 # FIXME:this is short term fix
101 main.log.info( "Removing raft logs" )
102 main.ONOSbench.onosRemoveRaftLogs()
103
104 main.log.info( "Uninstalling ONOS" )
Jon Halle1a3b752015-07-22 13:02:46 -0700105 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700106 main.ONOSbench.onosUninstall( node.ip_address )
107
108 # Make sure ONOS is DEAD
109 main.log.info( "Killing any ONOS processes" )
110 killResults = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -0700111 for node in main.nodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700112 killed = main.ONOSbench.onosKill( node.ip_address )
113 killResults = killResults and killed
114
115 cleanInstallResult = main.TRUE
116 gitPullResult = main.TRUE
117
118 main.step( "Starting Mininet" )
119 # scp topo file to mininet
120 # TODO: move to params?
121 topoName = "obelisk.py"
122 filePath = main.ONOSbench.home + "/tools/test/topos/"
kelvin-onlabd9e23de2015-08-06 10:34:44 -0700123 main.ONOSbench.scp( main.Mininet1,
124 filePath + topoName,
125 main.Mininet1.home,
126 direction="to" )
Jon Hall85794ff2015-07-08 14:12:30 -0700127 mnResult = main.Mininet1.startNet( )
128 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
129 onpass="Mininet Started",
130 onfail="Error starting Mininet" )
131
132 main.step( "Git checkout and pull " + gitBranch )
133 if PULLCODE:
134 main.ONOSbench.gitCheckout( gitBranch )
135 gitPullResult = main.ONOSbench.gitPull()
136 # values of 1 or 3 are good
137 utilities.assert_lesser( expect=0, actual=gitPullResult,
138 onpass="Git pull successful",
139 onfail="Git pull failed" )
140 main.ONOSbench.getVersion( report=True )
141
142 main.step( "Using mvn clean install" )
143 cleanInstallResult = main.TRUE
144 if PULLCODE and gitPullResult == main.TRUE:
145 cleanInstallResult = main.ONOSbench.cleanInstall()
146 else:
147 main.log.warn( "Did not pull new code so skipping mvn " +
148 "clean install" )
149 utilities.assert_equals( expect=main.TRUE,
150 actual=cleanInstallResult,
151 onpass="MCI successful",
152 onfail="MCI failed" )
153 # GRAPHS
154 # NOTE: important params here:
155 # job = name of Jenkins job
156 # Plot Name = Plot-HA, only can be used if multiple plots
157 # index = The number of the graph under plot name
Jon Hall5cf14d52015-07-16 12:15:19 -0700158 job = "HAsingleInstanceRestart"
Jon Hall85794ff2015-07-08 14:12:30 -0700159 plotName = "Plot-HA"
Jon Hall843f8bc2016-03-18 14:28:13 -0700160 index = "2"
Jon Hall85794ff2015-07-08 14:12:30 -0700161 graphs = '<ac:structured-macro ac:name="html">\n'
162 graphs += '<ac:plain-text-body><![CDATA[\n'
163 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
Jon Halla9845df2016-01-15 14:55:58 -0800164 '/plot/' + plotName + '/getPlot?index=' + index +\
Jon Hall85794ff2015-07-08 14:12:30 -0700165 '&width=500&height=300"' +\
166 'noborder="0" width="500" height="300" scrolling="yes" ' +\
167 'seamless="seamless"></iframe>\n'
168 graphs += ']]></ac:plain-text-body>\n'
169 graphs += '</ac:structured-macro>\n'
170 main.log.wiki(graphs)
171
Jon Halle1a3b752015-07-22 13:02:46 -0700172 main.CLIs = []
173 main.nodes = []
Jon Hall5cf14d52015-07-16 12:15:19 -0700174 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700175 for i in range( 1, main.numCtrls + 1 ):
176 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
177 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) )
178 ipList.append( main.nodes[ -1 ].ip_address )
Jon Hall5cf14d52015-07-16 12:15:19 -0700179
180 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, "SingleHA",
181 main.Mininet1.ip_address,
182 cellAppString, ipList[ 0 ] )
Jon Hall85794ff2015-07-08 14:12:30 -0700183 cellResult = main.ONOSbench.setCell( "SingleHA" )
184 verifyResult = main.ONOSbench.verifyCell()
185 main.step( "Creating ONOS package" )
186 packageResult = main.ONOSbench.onosPackage()
187 utilities.assert_equals( expect=main.TRUE, actual=packageResult,
188 onpass="ONOS package successful",
189 onfail="ONOS package failed" )
190
191 main.step( "Installing ONOS package" )
Jon Halla440e872016-03-31 15:15:50 -0700192 onosInstallResult = main.TRUE
193 for node in main.nodes:
194 tmpResult = main.ONOSbench.onosInstall( options="-f",
195 node=node.ip_address )
196 onosInstallResult = onosInstallResult and tmpResult
Jon Hall85794ff2015-07-08 14:12:30 -0700197 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
198 onpass="ONOS install successful",
199 onfail="ONOS install failed" )
200
201 main.step( "Checking if ONOS is up yet" )
202 for i in range( 2 ):
Jon Halla440e872016-03-31 15:15:50 -0700203 onosIsupResult = main.TRUE
204 for node in main.nodes:
205 started = main.ONOSbench.isup( node.ip_address )
206 if not started:
207 main.log.error( node.name + " hasn't started" )
208 onosIsupResult = onosIsupResult and started
209 if onosIsupResult == main.TRUE:
Jon Hall85794ff2015-07-08 14:12:30 -0700210 break
Jon Halla440e872016-03-31 15:15:50 -0700211 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult,
Jon Hall85794ff2015-07-08 14:12:30 -0700212 onpass="ONOS startup successful",
213 onfail="ONOS startup failed" )
214
Jon Hall6509dbf2016-06-21 17:01:17 -0700215 main.step( "Starting ONOS CLI sessions" )
Jon Halla440e872016-03-31 15:15:50 -0700216 cliResults = main.TRUE
217 threads = []
218 for i in range( main.numCtrls ):
219 t = main.Thread( target=main.CLIs[i].startOnosCli,
220 name="startOnosCli-" + str( i ),
221 args=[main.nodes[i].ip_address] )
222 threads.append( t )
223 t.start()
224
225 for t in threads:
226 t.join()
227 cliResults = cliResults and t.result
Jon Hall85794ff2015-07-08 14:12:30 -0700228 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
229 onpass="ONOS cli startup successful",
230 onfail="ONOS cli startup failed" )
231
Jon Halla440e872016-03-31 15:15:50 -0700232 # Create a list of active nodes for use when some nodes are stopped
233 main.activeNodes = [ i for i in range( 0, len( main.CLIs ) ) ]
234
Jon Hall85794ff2015-07-08 14:12:30 -0700235 if main.params[ 'tcpdump' ].lower() == "true":
236 main.step( "Start Packet Capture MN" )
237 main.Mininet2.startTcpdump(
238 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
239 + "-MN.pcap",
240 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
241 port=main.params[ 'MNtcpdump' ][ 'port' ] )
242
Jon Halla440e872016-03-31 15:15:50 -0700243 main.step( "Checking ONOS nodes" )
Jon Hall41d39f12016-04-11 22:54:35 -0700244 nodeResults = utilities.retry( main.HA.nodesCheck,
245 False,
246 args=[main.activeNodes],
247 attempts=5 )
Jon Halla440e872016-03-31 15:15:50 -0700248
Jon Hall41d39f12016-04-11 22:54:35 -0700249 utilities.assert_equals( expect=True, actual=nodeResults,
Jon Halla440e872016-03-31 15:15:50 -0700250 onpass="Nodes check successful",
251 onfail="Nodes check NOT successful" )
252
253 if not nodeResults:
Jon Hall7ac7bc32016-05-05 10:57:02 -0700254 for i in main.activeNodes:
255 cli = main.CLIs[i]
Jon Halla440e872016-03-31 15:15:50 -0700256 main.log.debug( "{} components not ACTIVE: \n{}".format(
257 cli.name,
258 cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700259 main.log.error( "Failed to start ONOS, stopping test" )
260 main.cleanup()
261 main.exit()
262
Jon Hall172b7ba2016-04-07 18:12:20 -0700263 main.step( "Activate apps defined in the params file" )
264 # get data from the params
265 apps = main.params.get( 'apps' )
266 if apps:
267 apps = apps.split(',')
268 main.log.warn( apps )
269 activateResult = True
270 for app in apps:
271 main.CLIs[ 0 ].app( app, "Activate" )
272 # TODO: check this worked
273 time.sleep( 10 ) # wait for apps to activate
274 for app in apps:
275 state = main.CLIs[ 0 ].appStatus( app )
276 if state == "ACTIVE":
277 activateResult = activeResult and True
278 else:
279 main.log.error( "{} is in {} state".format( app, state ) )
280 activeResult = False
281 utilities.assert_equals( expect=True,
282 actual=activateResult,
283 onpass="Successfully activated apps",
284 onfail="Failed to activate apps" )
285 else:
286 main.log.warn( "No apps were specified to be loaded after startup" )
287
288 main.step( "Set ONOS configurations" )
289 config = main.params.get( 'ONOS_Configuration' )
290 if config:
291 main.log.debug( config )
292 checkResult = main.TRUE
293 for component in config:
294 for setting in config[component]:
295 value = config[component][setting]
296 check = main.CLIs[ 0 ].setCfg( component, setting, value )
297 main.log.info( "Value was changed? {}".format( main.TRUE == check ) )
298 checkResult = check and checkResult
299 utilities.assert_equals( expect=main.TRUE,
300 actual=checkResult,
301 onpass="Successfully set config",
302 onfail="Failed to set config" )
303 else:
304 main.log.warn( "No configurations were specified to be changed after startup" )
305
Jon Hall9d2dcad2016-04-08 10:15:20 -0700306 main.step( "App Ids check" )
307 appCheck = main.TRUE
308 threads = []
309 for i in main.activeNodes:
310 t = main.Thread( target=main.CLIs[i].appToIDCheck,
311 name="appToIDCheck-" + str( i ),
312 args=[] )
313 threads.append( t )
314 t.start()
315
316 for t in threads:
317 t.join()
318 appCheck = appCheck and t.result
319 if appCheck != main.TRUE:
320 node = main.activeNodes[0]
321 main.log.warn( main.CLIs[node].apps() )
322 main.log.warn( main.CLIs[node].appIDs() )
323 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
324 onpass="App Ids seem to be correct",
325 onfail="Something is wrong with app Ids" )
326
Jon Hall85794ff2015-07-08 14:12:30 -0700327 def CASE2( self, main ):
328 """
329 Assign devices to controllers
330 """
331 import re
Jon Halle1a3b752015-07-22 13:02:46 -0700332 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700333 assert main, "main not defined"
334 assert utilities.assert_equals, "utilities.assert_equals not defined"
335
336 main.case( "Assigning devices to controllers" )
Jon Hall783bbf92015-07-23 14:33:19 -0700337 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700338 "and check that an ONOS node becomes the " +\
339 "master of the device."
340 main.step( "Assign switches to controllers" )
341
342 ipList = []
Jon Halle1a3b752015-07-22 13:02:46 -0700343 for i in range( main.numCtrls ):
344 ipList.append( main.nodes[ i ].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -0700345 swList = []
346 for i in range( 1, 29 ):
347 swList.append( "s" + str( i ) )
348 main.Mininet1.assignSwController( sw=swList, ip=ipList )
349
350 mastershipCheck = main.TRUE
351 for i in range( 1, 29 ):
352 response = main.Mininet1.getSwController( "s" + str( i ) )
353 try:
354 main.log.info( str( response ) )
355 except Exception:
356 main.log.info( repr( response ) )
Jon Halla440e872016-03-31 15:15:50 -0700357 for node in main.nodes:
358 if re.search( "tcp:" + node.ip_address, response ):
359 mastershipCheck = mastershipCheck and main.TRUE
360 else:
361 main.log.error( "Error, node " + node.ip_address + " is " +
362 "not in the list of controllers s" +
363 str( i ) + " is connecting to." )
364 mastershipCheck = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700365 utilities.assert_equals(
366 expect=main.TRUE,
367 actual=mastershipCheck,
368 onpass="Switch mastership assigned correctly",
369 onfail="Switches not assigned correctly to controllers" )
370
371 def CASE21( self, main ):
372 """
373 Assign mastership to controllers
374 """
Jon Halle1a3b752015-07-22 13:02:46 -0700375 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700376 assert main, "main not defined"
377 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -0700378 assert main.CLIs, "main.CLIs not defined"
379 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700380
381 main.case( "Assigning Controller roles for switches" )
Jon Hall783bbf92015-07-23 14:33:19 -0700382 main.caseExplanation = "Check that ONOS is connected to each " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700383 "device. Then manually assign" +\
384 " mastership to specific ONOS nodes using" +\
385 " 'device-role'"
386 main.step( "Assign mastership of switches to specific controllers" )
Jon Halla440e872016-03-31 15:15:50 -0700387 # Manually assign mastership to the controller we want
Jon Hall85794ff2015-07-08 14:12:30 -0700388 roleCall = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -0700389
390 ipList = [ ]
391 deviceList = []
392 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall85794ff2015-07-08 14:12:30 -0700393 try:
Jon Halla440e872016-03-31 15:15:50 -0700394 # Assign mastership to specific controllers. This assignment was
395 # determined for a 7 node cluser, but will work with any sized
396 # cluster
Jon Hall85794ff2015-07-08 14:12:30 -0700397 for i in range( 1, 29 ): # switches 1 through 28
Jon Hall85794ff2015-07-08 14:12:30 -0700398 # set up correct variables:
399 if i == 1:
Jon Halla440e872016-03-31 15:15:50 -0700400 c = 0
401 ip = main.nodes[ c ].ip_address # ONOS1
402 deviceId = onosCli.getDevice( "1000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700403 elif i == 2:
Jon Halla440e872016-03-31 15:15:50 -0700404 c = 1 % main.numCtrls
405 ip = main.nodes[ c ].ip_address # ONOS2
406 deviceId = onosCli.getDevice( "2000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700407 elif i == 3:
Jon Halla440e872016-03-31 15:15:50 -0700408 c = 1 % main.numCtrls
409 ip = main.nodes[ c ].ip_address # ONOS2
410 deviceId = onosCli.getDevice( "3000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700411 elif i == 4:
Jon Halla440e872016-03-31 15:15:50 -0700412 c = 3 % main.numCtrls
413 ip = main.nodes[ c ].ip_address # ONOS4
414 deviceId = onosCli.getDevice( "3004" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700415 elif i == 5:
Jon Halla440e872016-03-31 15:15:50 -0700416 c = 2 % main.numCtrls
417 ip = main.nodes[ c ].ip_address # ONOS3
418 deviceId = onosCli.getDevice( "5000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700419 elif i == 6:
Jon Halla440e872016-03-31 15:15:50 -0700420 c = 2 % main.numCtrls
421 ip = main.nodes[ c ].ip_address # ONOS3
422 deviceId = onosCli.getDevice( "6000" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700423 elif i == 7:
Jon Halla440e872016-03-31 15:15:50 -0700424 c = 5 % main.numCtrls
425 ip = main.nodes[ c ].ip_address # ONOS6
426 deviceId = onosCli.getDevice( "6007" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700427 elif i >= 8 and i <= 17:
Jon Halla440e872016-03-31 15:15:50 -0700428 c = 4 % main.numCtrls
429 ip = main.nodes[ c ].ip_address # ONOS5
Jon Hall85794ff2015-07-08 14:12:30 -0700430 dpid = '3' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700431 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700432 elif i >= 18 and i <= 27:
Jon Halla440e872016-03-31 15:15:50 -0700433 c = 6 % main.numCtrls
434 ip = main.nodes[ c ].ip_address # ONOS7
Jon Hall85794ff2015-07-08 14:12:30 -0700435 dpid = '6' + str( i ).zfill( 3 )
Jon Halla440e872016-03-31 15:15:50 -0700436 deviceId = onosCli.getDevice( dpid ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700437 elif i == 28:
Jon Halla440e872016-03-31 15:15:50 -0700438 c = 0
439 ip = main.nodes[ c ].ip_address # ONOS1
440 deviceId = onosCli.getDevice( "2800" ).get( 'id' )
Jon Hall85794ff2015-07-08 14:12:30 -0700441 else:
442 main.log.error( "You didn't write an else statement for " +
443 "switch s" + str( i ) )
Jon Halla440e872016-03-31 15:15:50 -0700444 roleCall = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -0700445 # Assign switch
446 assert deviceId, "No device id for s" + str( i ) + " in ONOS"
447 # TODO: make this controller dynamic
Jon Halla440e872016-03-31 15:15:50 -0700448 roleCall = roleCall and onosCli.deviceRole( deviceId, ip )
449 ipList.append( ip )
450 deviceList.append( deviceId )
Jon Hall85794ff2015-07-08 14:12:30 -0700451 except ( AttributeError, AssertionError ):
452 main.log.exception( "Something is wrong with ONOS device view" )
Jon Halla440e872016-03-31 15:15:50 -0700453 main.log.info( onosCli.devices() )
Jon Hall85794ff2015-07-08 14:12:30 -0700454 utilities.assert_equals(
455 expect=main.TRUE,
456 actual=roleCall,
457 onpass="Re-assigned switch mastership to designated controller",
458 onfail="Something wrong with deviceRole calls" )
459
460 main.step( "Check mastership was correctly assigned" )
Jon Halla440e872016-03-31 15:15:50 -0700461 roleCheck = main.TRUE
462 # NOTE: This is due to the fact that device mastership change is not
463 # atomic and is actually a multi step process
464 time.sleep( 5 )
465 for i in range( len( ipList ) ):
466 ip = ipList[i]
467 deviceId = deviceList[i]
468 # Check assignment
469 master = onosCli.getRole( deviceId ).get( 'master' )
470 if ip in master:
471 roleCheck = roleCheck and main.TRUE
472 else:
473 roleCheck = roleCheck and main.FALSE
474 main.log.error( "Error, controller " + ip + " is not" +
475 " master " + "of device " +
476 str( deviceId ) + ". Master is " +
477 repr( master ) + "." )
Jon Hall85794ff2015-07-08 14:12:30 -0700478 utilities.assert_equals(
479 expect=main.TRUE,
480 actual=roleCheck,
481 onpass="Switches were successfully reassigned to designated " +
482 "controller",
483 onfail="Switches were not successfully reassigned" )
484
485 def CASE3( self, main ):
486 """
487 Assign intents
488 """
489 import time
490 import json
Jon Halle1a3b752015-07-22 13:02:46 -0700491 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700492 assert main, "main not defined"
493 assert utilities.assert_equals, "utilities.assert_equals not defined"
494 # NOTE: we must reinstall intents until we have a persistant intent
495 # datastore!
496 main.case( "Adding host Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700497 main.caseExplanation = "Discover hosts by using pingall then " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700498 "assign predetermined host-to-host intents." +\
499 " After installation, check that the intent" +\
500 " is distributed to all nodes and the state" +\
501 " is INSTALLED"
502
503 # install onos-app-fwd
504 main.step( "Install reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700505 onosCli = main.CLIs[ main.activeNodes[0] ]
506 installResults = onosCli.activateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700507 utilities.assert_equals( expect=main.TRUE, actual=installResults,
508 onpass="Install fwd successful",
509 onfail="Install fwd failed" )
510
511 main.step( "Check app ids" )
Jon Halla440e872016-03-31 15:15:50 -0700512 appCheck = main.TRUE
513 threads = []
514 for i in main.activeNodes:
515 t = main.Thread( target=main.CLIs[i].appToIDCheck,
516 name="appToIDCheck-" + str( i ),
517 args=[] )
518 threads.append( t )
519 t.start()
520
521 for t in threads:
522 t.join()
523 appCheck = appCheck and t.result
Jon Hall85794ff2015-07-08 14:12:30 -0700524 if appCheck != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700525 main.log.warn( onosCli.apps() )
526 main.log.warn( onosCli.appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700527 utilities.assert_equals( expect=main.TRUE, actual=appCheck,
528 onpass="App Ids seem to be correct",
529 onfail="Something is wrong with app Ids" )
530
531 main.step( "Discovering Hosts( Via pingall for now )" )
532 # FIXME: Once we have a host discovery mechanism, use that instead
533 # REACTIVE FWD test
534 pingResult = main.FALSE
Jon Hall96091e62015-09-21 17:34:17 -0700535 passMsg = "Reactive Pingall test passed"
536 time1 = time.time()
537 pingResult = main.Mininet1.pingall()
538 time2 = time.time()
539 if not pingResult:
540 main.log.warn("First pingall failed. Trying again...")
Jon Hall85794ff2015-07-08 14:12:30 -0700541 pingResult = main.Mininet1.pingall()
Jon Hall96091e62015-09-21 17:34:17 -0700542 passMsg += " on the second try"
543 utilities.assert_equals(
544 expect=main.TRUE,
545 actual=pingResult,
546 onpass= passMsg,
547 onfail="Reactive Pingall failed, " +
548 "one or more ping pairs failed" )
549 main.log.info( "Time for pingall: %2f seconds" %
550 ( time2 - time1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700551 # timeout for fwd flows
552 time.sleep( 11 )
553 # uninstall onos-app-fwd
554 main.step( "Uninstall reactive forwarding app" )
Jon Halla440e872016-03-31 15:15:50 -0700555 node = main.activeNodes[0]
556 uninstallResult = main.CLIs[node].deactivateApp( "org.onosproject.fwd" )
Jon Hall85794ff2015-07-08 14:12:30 -0700557 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
558 onpass="Uninstall fwd successful",
559 onfail="Uninstall fwd failed" )
560
561 main.step( "Check app ids" )
Jon Halla440e872016-03-31 15:15:50 -0700562 threads = []
563 appCheck2 = main.TRUE
564 for i in main.activeNodes:
565 t = main.Thread( target=main.CLIs[i].appToIDCheck,
566 name="appToIDCheck-" + str( i ),
567 args=[] )
568 threads.append( t )
569 t.start()
570
571 for t in threads:
572 t.join()
573 appCheck2 = appCheck2 and t.result
Jon Hall85794ff2015-07-08 14:12:30 -0700574 if appCheck2 != main.TRUE:
Jon Halla440e872016-03-31 15:15:50 -0700575 node = main.activeNodes[0]
576 main.log.warn( main.CLIs[node].apps() )
577 main.log.warn( main.CLIs[node].appIDs() )
Jon Hall85794ff2015-07-08 14:12:30 -0700578 utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
579 onpass="App Ids seem to be correct",
580 onfail="Something is wrong with app Ids" )
581
582 main.step( "Add host intents via cli" )
583 intentIds = []
Jon Halla440e872016-03-31 15:15:50 -0700584 # TODO: move the host numbers to params
585 # Maybe look at all the paths we ping?
Jon Hall85794ff2015-07-08 14:12:30 -0700586 intentAddResult = True
587 hostResult = main.TRUE
588 for i in range( 8, 18 ):
589 main.log.info( "Adding host intent between h" + str( i ) +
590 " and h" + str( i + 10 ) )
591 host1 = "00:00:00:00:00:" + \
592 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
593 host2 = "00:00:00:00:00:" + \
594 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
595 # NOTE: getHost can return None
Jon Halla440e872016-03-31 15:15:50 -0700596 host1Dict = onosCli.getHost( host1 )
597 host2Dict = onosCli.getHost( host2 )
Jon Hall85794ff2015-07-08 14:12:30 -0700598 host1Id = None
599 host2Id = None
600 if host1Dict and host2Dict:
601 host1Id = host1Dict.get( 'id', None )
602 host2Id = host2Dict.get( 'id', None )
603 if host1Id and host2Id:
Jon Halla440e872016-03-31 15:15:50 -0700604 nodeNum = ( i % len( main.activeNodes ) )
605 node = main.activeNodes[nodeNum]
606 tmpId = main.CLIs[node].addHostIntent( host1Id, host2Id )
Jon Hall85794ff2015-07-08 14:12:30 -0700607 if tmpId:
608 main.log.info( "Added intent with id: " + tmpId )
609 intentIds.append( tmpId )
610 else:
611 main.log.error( "addHostIntent returned: " +
612 repr( tmpId ) )
613 else:
614 main.log.error( "Error, getHost() failed for h" + str( i ) +
615 " and/or h" + str( i + 10 ) )
Jon Halla440e872016-03-31 15:15:50 -0700616 node = main.activeNodes[0]
617 hosts = main.CLIs[node].hosts()
Jon Hall85794ff2015-07-08 14:12:30 -0700618 main.log.warn( "Hosts output: " )
619 try:
620 main.log.warn( json.dumps( json.loads( hosts ),
621 sort_keys=True,
622 indent=4,
623 separators=( ',', ': ' ) ) )
624 except ( ValueError, TypeError ):
625 main.log.warn( repr( hosts ) )
626 hostResult = main.FALSE
627 utilities.assert_equals( expect=main.TRUE, actual=hostResult,
628 onpass="Found a host id for each host",
629 onfail="Error looking up host ids" )
630
631 intentStart = time.time()
Jon Halla440e872016-03-31 15:15:50 -0700632 onosIds = onosCli.getAllIntentsId()
Jon Hall85794ff2015-07-08 14:12:30 -0700633 main.log.info( "Submitted intents: " + str( intentIds ) )
634 main.log.info( "Intents in ONOS: " + str( onosIds ) )
635 for intent in intentIds:
636 if intent in onosIds:
637 pass # intent submitted is in onos
638 else:
639 intentAddResult = False
640 if intentAddResult:
641 intentStop = time.time()
642 else:
643 intentStop = None
644 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700645 intents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -0700646 intentStates = []
647 installedCheck = True
648 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
649 count = 0
650 try:
651 for intent in json.loads( intents ):
652 state = intent.get( 'state', None )
653 if "INSTALLED" not in state:
654 installedCheck = False
655 intentId = intent.get( 'id', None )
656 intentStates.append( ( intentId, state ) )
657 except ( ValueError, TypeError ):
658 main.log.exception( "Error parsing intents" )
659 # add submitted intents not in the store
660 tmplist = [ i for i, s in intentStates ]
661 missingIntents = False
662 for i in intentIds:
663 if i not in tmplist:
664 intentStates.append( ( i, " - " ) )
665 missingIntents = True
666 intentStates.sort()
667 for i, s in intentStates:
668 count += 1
669 main.log.info( "%-6s%-15s%-15s" %
670 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700671 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -0700672 try:
673 missing = False
674 if leaders:
675 parsedLeaders = json.loads( leaders )
676 main.log.warn( json.dumps( parsedLeaders,
677 sort_keys=True,
678 indent=4,
679 separators=( ',', ': ' ) ) )
680 # check for all intent partitions
681 topics = []
682 for i in range( 14 ):
683 topics.append( "intent-partition-" + str( i ) )
684 main.log.debug( topics )
685 ONOStopics = [ j['topic'] for j in parsedLeaders ]
686 for topic in topics:
687 if topic not in ONOStopics:
688 main.log.error( "Error: " + topic +
689 " not in leaders" )
690 missing = True
691 else:
692 main.log.error( "leaders() returned None" )
693 except ( ValueError, TypeError ):
694 main.log.exception( "Error parsing leaders" )
695 main.log.error( repr( leaders ) )
696 # Check all nodes
697 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700698 for i in main.activeNodes:
699 response = main.CLIs[i].leaders( jsonFormat=False)
700 main.log.warn( str( main.CLIs[i].name ) + " leaders output: \n" +
701 str( response ) )
Jon Hall85794ff2015-07-08 14:12:30 -0700702
Jon Halla440e872016-03-31 15:15:50 -0700703 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -0700704 try:
705 if partitions :
706 parsedPartitions = json.loads( partitions )
707 main.log.warn( json.dumps( parsedPartitions,
708 sort_keys=True,
709 indent=4,
710 separators=( ',', ': ' ) ) )
711 # TODO check for a leader in all paritions
712 # TODO check for consistency among nodes
713 else:
714 main.log.error( "partitions() returned None" )
715 except ( ValueError, TypeError ):
716 main.log.exception( "Error parsing partitions" )
717 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700718 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -0700719 try:
720 if pendingMap :
721 parsedPending = json.loads( pendingMap )
722 main.log.warn( json.dumps( parsedPending,
723 sort_keys=True,
724 indent=4,
725 separators=( ',', ': ' ) ) )
726 # TODO check something here?
727 else:
728 main.log.error( "pendingMap() returned None" )
729 except ( ValueError, TypeError ):
730 main.log.exception( "Error parsing pending map" )
731 main.log.error( repr( pendingMap ) )
732
733 intentAddResult = bool( intentAddResult and not missingIntents and
734 installedCheck )
735 if not intentAddResult:
736 main.log.error( "Error in pushing host intents to ONOS" )
737
738 main.step( "Intent Anti-Entropy dispersion" )
Jon Halla440e872016-03-31 15:15:50 -0700739 for j in range(100):
Jon Hall85794ff2015-07-08 14:12:30 -0700740 correct = True
741 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700742 for i in main.activeNodes:
Jon Hall85794ff2015-07-08 14:12:30 -0700743 onosIds = []
Jon Halla440e872016-03-31 15:15:50 -0700744 ids = main.CLIs[i].getAllIntentsId()
Jon Hall85794ff2015-07-08 14:12:30 -0700745 onosIds.append( ids )
Jon Halla440e872016-03-31 15:15:50 -0700746 main.log.debug( "Intents in " + main.CLIs[i].name + ": " +
Jon Hall85794ff2015-07-08 14:12:30 -0700747 str( sorted( onosIds ) ) )
748 if sorted( ids ) != sorted( intentIds ):
749 main.log.warn( "Set of intent IDs doesn't match" )
750 correct = False
751 break
752 else:
Jon Halla440e872016-03-31 15:15:50 -0700753 intents = json.loads( main.CLIs[i].intents() )
Jon Hall85794ff2015-07-08 14:12:30 -0700754 for intent in intents:
755 if intent[ 'state' ] != "INSTALLED":
756 main.log.warn( "Intent " + intent[ 'id' ] +
757 " is " + intent[ 'state' ] )
758 correct = False
759 break
760 if correct:
761 break
762 else:
763 time.sleep(1)
764 if not intentStop:
765 intentStop = time.time()
766 global gossipTime
767 gossipTime = intentStop - intentStart
768 main.log.info( "It took about " + str( gossipTime ) +
769 " seconds for all intents to appear in each node" )
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700770 gossipPeriod = int( main.params['timers']['gossip'] )
Jon Halla440e872016-03-31 15:15:50 -0700771 maxGossipTime = gossipPeriod * len( main.activeNodes )
Jon Hall85794ff2015-07-08 14:12:30 -0700772 utilities.assert_greater_equals(
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700773 expect=maxGossipTime, actual=gossipTime,
Jon Hall85794ff2015-07-08 14:12:30 -0700774 onpass="ECM anti-entropy for intents worked within " +
775 "expected time",
Jon Hallb3ed8ed2015-10-28 16:43:55 -0700776 onfail="Intent ECM anti-entropy took too long. " +
777 "Expected time:{}, Actual time:{}".format( maxGossipTime,
778 gossipTime ) )
779 if gossipTime <= maxGossipTime:
Jon Hall85794ff2015-07-08 14:12:30 -0700780 intentAddResult = True
781
782 if not intentAddResult or "key" in pendingMap:
783 import time
784 installedCheck = True
785 main.log.info( "Sleeping 60 seconds to see if intents are found" )
786 time.sleep( 60 )
Jon Halla440e872016-03-31 15:15:50 -0700787 onosIds = onosCli.getAllIntentsId()
Jon Hall85794ff2015-07-08 14:12:30 -0700788 main.log.info( "Submitted intents: " + str( intentIds ) )
789 main.log.info( "Intents in ONOS: " + str( onosIds ) )
790 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -0700791 intents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -0700792 intentStates = []
793 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
794 count = 0
795 try:
796 for intent in json.loads( intents ):
797 # Iter through intents of a node
798 state = intent.get( 'state', None )
799 if "INSTALLED" not in state:
800 installedCheck = False
801 intentId = intent.get( 'id', None )
802 intentStates.append( ( intentId, state ) )
803 except ( ValueError, TypeError ):
804 main.log.exception( "Error parsing intents" )
805 # add submitted intents not in the store
806 tmplist = [ i for i, s in intentStates ]
807 for i in intentIds:
808 if i not in tmplist:
809 intentStates.append( ( i, " - " ) )
810 intentStates.sort()
811 for i, s in intentStates:
812 count += 1
813 main.log.info( "%-6s%-15s%-15s" %
814 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -0700815 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -0700816 try:
817 missing = False
818 if leaders:
819 parsedLeaders = json.loads( leaders )
820 main.log.warn( json.dumps( parsedLeaders,
821 sort_keys=True,
822 indent=4,
823 separators=( ',', ': ' ) ) )
824 # check for all intent partitions
825 # check for election
826 topics = []
827 for i in range( 14 ):
828 topics.append( "intent-partition-" + str( i ) )
829 # FIXME: this should only be after we start the app
830 topics.append( "org.onosproject.election" )
831 main.log.debug( topics )
832 ONOStopics = [ j['topic'] for j in parsedLeaders ]
833 for topic in topics:
834 if topic not in ONOStopics:
835 main.log.error( "Error: " + topic +
836 " not in leaders" )
837 missing = True
838 else:
839 main.log.error( "leaders() returned None" )
840 except ( ValueError, TypeError ):
841 main.log.exception( "Error parsing leaders" )
842 main.log.error( repr( leaders ) )
843 # Check all nodes
844 if missing:
Jon Halla440e872016-03-31 15:15:50 -0700845 for i in main.activeNodes:
846 node = main.CLIs[i]
847 response = node.leaders( jsonFormat=False)
848 main.log.warn( str( node.name ) + " leaders output: \n" +
849 str( response ) )
850
851 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -0700852 try:
853 if partitions :
854 parsedPartitions = json.loads( partitions )
855 main.log.warn( json.dumps( parsedPartitions,
856 sort_keys=True,
857 indent=4,
858 separators=( ',', ': ' ) ) )
859 # TODO check for a leader in all paritions
860 # TODO check for consistency among nodes
861 else:
862 main.log.error( "partitions() returned None" )
863 except ( ValueError, TypeError ):
864 main.log.exception( "Error parsing partitions" )
865 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -0700866 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -0700867 try:
868 if pendingMap :
869 parsedPending = json.loads( pendingMap )
870 main.log.warn( json.dumps( parsedPending,
871 sort_keys=True,
872 indent=4,
873 separators=( ',', ': ' ) ) )
874 # TODO check something here?
875 else:
876 main.log.error( "pendingMap() returned None" )
877 except ( ValueError, TypeError ):
878 main.log.exception( "Error parsing pending map" )
879 main.log.error( repr( pendingMap ) )
880
881 def CASE4( self, main ):
882 """
883 Ping across added host intents
884 """
885 import json
886 import time
Jon Halle1a3b752015-07-22 13:02:46 -0700887 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -0700888 assert main, "main not defined"
889 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halla440e872016-03-31 15:15:50 -0700890 main.case( "Verify connectivity by sending traffic across Intents" )
Jon Hall783bbf92015-07-23 14:33:19 -0700891 main.caseExplanation = "Ping across added host intents to check " +\
Jon Hall85794ff2015-07-08 14:12:30 -0700892 "functionality and check the state of " +\
893 "the intent"
Jon Hall9d2dcad2016-04-08 10:15:20 -0700894
Jon Hall41d39f12016-04-11 22:54:35 -0700895 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall9d2dcad2016-04-08 10:15:20 -0700896 main.step( "Check Intent state" )
897 installedCheck = True
898 # Print the intent states
899 intents = main.ONOScli1.intents()
900 intentStates = []
901 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
902 count = 0
903 # Iter through intents of a node
904 try:
905 for intent in json.loads( intents ):
906 state = intent.get( 'state', None )
907 if "INSTALLED" not in state:
908 installedCheck = False
909 intentId = intent.get( 'id', None )
910 intentStates.append( ( intentId, state ) )
911 except ( ValueError, TypeError ):
912 main.log.exception( "Error parsing intents." )
913 # Print states
914 intentStates.sort()
915 for i, s in intentStates:
916 count += 1
917 main.log.info( "%-6s%-15s%-15s" %
918 ( str( count ), str( i ), str( s ) ) )
919 utilities.assert_equals( expect=True, actual=installedCheck,
920 onpass="Intents are all INSTALLED",
921 onfail="Intents are not all in " +
922 "INSTALLED state" )
923
Jon Hall85794ff2015-07-08 14:12:30 -0700924 main.step( "Ping across added host intents" )
925 PingResult = main.TRUE
926 for i in range( 8, 18 ):
927 ping = main.Mininet1.pingHost( src="h" + str( i ),
928 target="h" + str( i + 10 ) )
929 PingResult = PingResult and ping
930 if ping == main.FALSE:
931 main.log.warn( "Ping failed between h" + str( i ) +
932 " and h" + str( i + 10 ) )
933 elif ping == main.TRUE:
934 main.log.info( "Ping test passed!" )
935 # Don't set PingResult or you'd override failures
936 if PingResult == main.FALSE:
937 main.log.error(
938 "Intents have not been installed correctly, pings failed." )
939 # TODO: pretty print
940 main.log.warn( "ONOS1 intents: " )
941 try:
Jon Halla440e872016-03-31 15:15:50 -0700942 tmpIntents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -0700943 main.log.warn( json.dumps( json.loads( tmpIntents ),
944 sort_keys=True,
945 indent=4,
946 separators=( ',', ': ' ) ) )
947 except ( ValueError, TypeError ):
948 main.log.warn( repr( tmpIntents ) )
949 utilities.assert_equals(
950 expect=main.TRUE,
951 actual=PingResult,
952 onpass="Intents have been installed correctly and pings work",
953 onfail="Intents have not been installed correctly, pings failed." )
954
Jon Hall85794ff2015-07-08 14:12:30 -0700955 main.step( "Check leadership of topics" )
Jon Halla440e872016-03-31 15:15:50 -0700956 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -0700957 topicCheck = main.TRUE
958 try:
959 if leaders:
960 parsedLeaders = json.loads( leaders )
961 main.log.warn( json.dumps( parsedLeaders,
962 sort_keys=True,
963 indent=4,
964 separators=( ',', ': ' ) ) )
965 # check for all intent partitions
966 # check for election
967 # TODO: Look at Devices as topics now that it uses this system
968 topics = []
969 for i in range( 14 ):
970 topics.append( "intent-partition-" + str( i ) )
971 # FIXME: this should only be after we start the app
972 # FIXME: topics.append( "org.onosproject.election" )
973 # Print leaders output
974 main.log.debug( topics )
975 ONOStopics = [ j['topic'] for j in parsedLeaders ]
976 for topic in topics:
977 if topic not in ONOStopics:
978 main.log.error( "Error: " + topic +
979 " not in leaders" )
980 topicCheck = main.FALSE
981 else:
982 main.log.error( "leaders() returned None" )
983 topicCheck = main.FALSE
984 except ( ValueError, TypeError ):
985 topicCheck = main.FALSE
986 main.log.exception( "Error parsing leaders" )
987 main.log.error( repr( leaders ) )
988 # TODO: Check for a leader of these topics
Jon Halla440e872016-03-31 15:15:50 -0700989 # Check all nodes
990 if topicCheck:
991 for i in main.activeNodes:
992 node = main.CLIs[i]
993 response = node.leaders( jsonFormat=False)
994 main.log.warn( str( node.name ) + " leaders output: \n" +
995 str( response ) )
996
Jon Hall85794ff2015-07-08 14:12:30 -0700997 utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
998 onpass="intent Partitions is in leaders",
999 onfail="Some topics were lost " )
1000 # Print partitions
Jon Halla440e872016-03-31 15:15:50 -07001001 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -07001002 try:
1003 if partitions :
1004 parsedPartitions = json.loads( partitions )
1005 main.log.warn( json.dumps( parsedPartitions,
1006 sort_keys=True,
1007 indent=4,
1008 separators=( ',', ': ' ) ) )
1009 # TODO check for a leader in all paritions
1010 # TODO check for consistency among nodes
1011 else:
1012 main.log.error( "partitions() returned None" )
1013 except ( ValueError, TypeError ):
1014 main.log.exception( "Error parsing partitions" )
1015 main.log.error( repr( partitions ) )
1016 # Print Pending Map
Jon Halla440e872016-03-31 15:15:50 -07001017 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -07001018 try:
1019 if pendingMap :
1020 parsedPending = json.loads( pendingMap )
1021 main.log.warn( json.dumps( parsedPending,
1022 sort_keys=True,
1023 indent=4,
1024 separators=( ',', ': ' ) ) )
1025 # TODO check something here?
1026 else:
1027 main.log.error( "pendingMap() returned None" )
1028 except ( ValueError, TypeError ):
1029 main.log.exception( "Error parsing pending map" )
1030 main.log.error( repr( pendingMap ) )
1031
1032 if not installedCheck:
1033 main.log.info( "Waiting 60 seconds to see if the state of " +
1034 "intents change" )
1035 time.sleep( 60 )
1036 # Print the intent states
Jon Halla440e872016-03-31 15:15:50 -07001037 intents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -07001038 intentStates = []
1039 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
1040 count = 0
1041 # Iter through intents of a node
1042 try:
1043 for intent in json.loads( intents ):
1044 state = intent.get( 'state', None )
1045 if "INSTALLED" not in state:
1046 installedCheck = False
1047 intentId = intent.get( 'id', None )
1048 intentStates.append( ( intentId, state ) )
1049 except ( ValueError, TypeError ):
1050 main.log.exception( "Error parsing intents." )
1051 intentStates.sort()
1052 for i, s in intentStates:
1053 count += 1
1054 main.log.info( "%-6s%-15s%-15s" %
1055 ( str( count ), str( i ), str( s ) ) )
Jon Halla440e872016-03-31 15:15:50 -07001056 leaders = onosCli.leaders()
Jon Hall85794ff2015-07-08 14:12:30 -07001057 try:
1058 missing = False
1059 if leaders:
1060 parsedLeaders = json.loads( leaders )
1061 main.log.warn( json.dumps( parsedLeaders,
1062 sort_keys=True,
1063 indent=4,
1064 separators=( ',', ': ' ) ) )
1065 # check for all intent partitions
1066 # check for election
1067 topics = []
1068 for i in range( 14 ):
1069 topics.append( "intent-partition-" + str( i ) )
1070 # FIXME: this should only be after we start the app
1071 topics.append( "org.onosproject.election" )
1072 main.log.debug( topics )
1073 ONOStopics = [ j['topic'] for j in parsedLeaders ]
1074 for topic in topics:
1075 if topic not in ONOStopics:
1076 main.log.error( "Error: " + topic +
1077 " not in leaders" )
1078 missing = True
1079 else:
1080 main.log.error( "leaders() returned None" )
1081 except ( ValueError, TypeError ):
1082 main.log.exception( "Error parsing leaders" )
1083 main.log.error( repr( leaders ) )
1084 if missing:
Jon Halla440e872016-03-31 15:15:50 -07001085 for i in main.activeNodes:
1086 node = main.CLIs[i]
1087 response = node.leaders( jsonFormat=False)
1088 main.log.warn( str( node.name ) + " leaders output: \n" +
1089 str( response ) )
1090
1091 partitions = onosCli.partitions()
Jon Hall85794ff2015-07-08 14:12:30 -07001092 try:
1093 if partitions :
1094 parsedPartitions = json.loads( partitions )
1095 main.log.warn( json.dumps( parsedPartitions,
1096 sort_keys=True,
1097 indent=4,
1098 separators=( ',', ': ' ) ) )
1099 # TODO check for a leader in all paritions
1100 # TODO check for consistency among nodes
1101 else:
1102 main.log.error( "partitions() returned None" )
1103 except ( ValueError, TypeError ):
1104 main.log.exception( "Error parsing partitions" )
1105 main.log.error( repr( partitions ) )
Jon Halla440e872016-03-31 15:15:50 -07001106 pendingMap = onosCli.pendingMap()
Jon Hall85794ff2015-07-08 14:12:30 -07001107 try:
1108 if pendingMap :
1109 parsedPending = json.loads( pendingMap )
1110 main.log.warn( json.dumps( parsedPending,
1111 sort_keys=True,
1112 indent=4,
1113 separators=( ',', ': ' ) ) )
1114 # TODO check something here?
1115 else:
1116 main.log.error( "pendingMap() returned None" )
1117 except ( ValueError, TypeError ):
1118 main.log.exception( "Error parsing pending map" )
1119 main.log.error( repr( pendingMap ) )
1120 # Print flowrules
Jon Halla440e872016-03-31 15:15:50 -07001121 node = main.activeNodes[0]
1122 main.log.debug( main.CLIs[node].flows( jsonFormat=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001123 main.step( "Wait a minute then ping again" )
1124 # the wait is above
1125 PingResult = main.TRUE
1126 for i in range( 8, 18 ):
1127 ping = main.Mininet1.pingHost( src="h" + str( i ),
1128 target="h" + str( i + 10 ) )
1129 PingResult = PingResult and ping
1130 if ping == main.FALSE:
1131 main.log.warn( "Ping failed between h" + str( i ) +
1132 " and h" + str( i + 10 ) )
1133 elif ping == main.TRUE:
1134 main.log.info( "Ping test passed!" )
1135 # Don't set PingResult or you'd override failures
1136 if PingResult == main.FALSE:
1137 main.log.error(
1138 "Intents have not been installed correctly, pings failed." )
1139 # TODO: pretty print
1140 main.log.warn( "ONOS1 intents: " )
1141 try:
Jon Halla440e872016-03-31 15:15:50 -07001142 tmpIntents = onosCli.intents()
Jon Hall85794ff2015-07-08 14:12:30 -07001143 main.log.warn( json.dumps( json.loads( tmpIntents ),
1144 sort_keys=True,
1145 indent=4,
1146 separators=( ',', ': ' ) ) )
1147 except ( ValueError, TypeError ):
1148 main.log.warn( repr( tmpIntents ) )
1149 utilities.assert_equals(
1150 expect=main.TRUE,
1151 actual=PingResult,
1152 onpass="Intents have been installed correctly and pings work",
1153 onfail="Intents have not been installed correctly, pings failed." )
1154
1155 def CASE5( self, main ):
1156 """
1157 Reading state of ONOS
1158 """
1159 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001160 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001161 assert main, "main not defined"
1162 assert utilities.assert_equals, "utilities.assert_equals not defined"
1163
1164 main.case( "Setting up and gathering data for current state" )
1165 # The general idea for this test case is to pull the state of
1166 # ( intents,flows, topology,... ) from each ONOS node
1167 # We can then compare them with each other and also with past states
1168
1169 main.step( "Check that each switch has a master" )
1170 global mastershipState
1171 mastershipState = '[]'
1172
1173 # Assert that each device has a master
Jon Halla440e872016-03-31 15:15:50 -07001174 rolesNotNull = main.TRUE
1175 threads = []
1176 for i in main.activeNodes:
1177 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1178 name="rolesNotNull-" + str( i ),
1179 args=[] )
1180 threads.append( t )
1181 t.start()
1182
1183 for t in threads:
1184 t.join()
1185 rolesNotNull = rolesNotNull and t.result
Jon Hall85794ff2015-07-08 14:12:30 -07001186 utilities.assert_equals(
1187 expect=main.TRUE,
1188 actual=rolesNotNull,
1189 onpass="Each device has a master",
1190 onfail="Some devices don't have a master assigned" )
1191
1192 main.step( "Get the Mastership of each switch" )
1193 ONOS1Mastership = main.ONOScli1.roles()
1194 # TODO: Make this a meaningful check
1195 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1196 main.log.error( "Error in getting ONOS roles" )
1197 main.log.warn(
1198 "ONOS1 mastership response: " +
1199 repr( ONOS1Mastership ) )
1200 consistentMastership = main.FALSE
1201 else:
1202 mastershipState = ONOS1Mastership
1203 consistentMastership = main.TRUE
1204
1205 main.step( "Get the intents from each controller" )
1206 global intentState
1207 intentState = []
1208 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1209 intentCheck = main.FALSE
1210 if "Error" in ONOS1Intents or not ONOS1Intents:
1211 main.log.error( "Error in getting ONOS intents" )
1212 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1213 else:
1214 intentCheck = main.TRUE
1215
1216 main.step( "Get the flows from each controller" )
1217 global flowState
1218 flowState = []
1219 flowCheck = main.FALSE
1220 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
1221 if "Error" in ONOS1Flows or not ONOS1Flows:
1222 main.log.error( "Error in getting ONOS flows" )
1223 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
1224 else:
1225 # TODO: Do a better check, maybe compare flows on switches?
1226 flowState = ONOS1Flows
1227 flowCheck = main.TRUE
1228
1229 main.step( "Get the OF Table entries" )
1230 global flows
1231 flows = []
1232 for i in range( 1, 29 ):
GlennRC68467eb2015-11-16 18:01:01 -08001233 flows.append( main.Mininet1.getFlowTable( "s" + str( i ), version="1.3", debug=False ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001234 if flowCheck == main.FALSE:
1235 for table in flows:
1236 main.log.warn( table )
1237 # TODO: Compare switch flow tables with ONOS flow tables
1238
1239 main.step( "Collecting topology information from ONOS" )
1240 devices = []
1241 devices.append( main.ONOScli1.devices() )
1242 hosts = []
1243 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1244 ports = []
1245 ports.append( main.ONOScli1.ports() )
1246 links = []
1247 links.append( main.ONOScli1.links() )
1248 clusters = []
1249 clusters.append( main.ONOScli1.clusters() )
1250
1251 main.step( "Each host has an IP address" )
1252 ipResult = main.TRUE
1253 for controller in range( 0, len( hosts ) ):
Jon Halla440e872016-03-31 15:15:50 -07001254 controllerStr = str( main.activeNodes[controller] + 1 )
1255 if hosts[ controller ]:
1256 for host in hosts[ controller ]:
1257 if not host.get( 'ipAddresses', [ ] ):
1258 main.log.error( "Error with host ips on controller" +
1259 controllerStr + ": " + str( host ) )
1260 ipResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07001261 utilities.assert_equals(
1262 expect=main.TRUE,
1263 actual=ipResult,
1264 onpass="The ips of the hosts aren't empty",
1265 onfail="The ip of at least one host is missing" )
1266
1267 # there should always only be one cluster
1268 main.step( "There is only one dataplane cluster" )
1269 try:
1270 numClusters = len( json.loads( clusters[ 0 ] ) )
1271 except ( ValueError, TypeError ):
1272 main.log.exception( "Error parsing clusters[0]: " +
1273 repr( clusters[ 0 ] ) )
Jon Hall6e709752016-02-01 13:38:46 -08001274 numClusters = "ERROR"
Jon Hall85794ff2015-07-08 14:12:30 -07001275 clusterResults = main.FALSE
1276 if numClusters == 1:
1277 clusterResults = main.TRUE
1278 utilities.assert_equals(
1279 expect=1,
1280 actual=numClusters,
1281 onpass="ONOS shows 1 SCC",
1282 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1283
1284 main.step( "Comparing ONOS topology to MN" )
1285 devicesResults = main.TRUE
1286 linksResults = main.TRUE
1287 hostsResults = main.TRUE
1288 mnSwitches = main.Mininet1.getSwitches()
1289 mnLinks = main.Mininet1.getLinks()
1290 mnHosts = main.Mininet1.getHosts()
Jon Halla440e872016-03-31 15:15:50 -07001291 for controller in main.activeNodes:
1292 controllerStr = str( main.activeNodes[controller] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07001293 if devices[ controller ] and ports[ controller ] and\
1294 "Error" not in devices[ controller ] and\
1295 "Error" not in ports[ controller ]:
Jon Hall6e709752016-02-01 13:38:46 -08001296 currentDevicesResult = main.Mininet1.compareSwitches(
1297 mnSwitches,
1298 json.loads( devices[ controller ] ),
1299 json.loads( ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001300 else:
1301 currentDevicesResult = main.FALSE
1302 utilities.assert_equals( expect=main.TRUE,
1303 actual=currentDevicesResult,
1304 onpass="ONOS" + controllerStr +
1305 " Switches view is correct",
1306 onfail="ONOS" + controllerStr +
1307 " Switches view is incorrect" )
1308 if links[ controller ] and "Error" not in links[ controller ]:
1309 currentLinksResult = main.Mininet1.compareLinks(
1310 mnSwitches, mnLinks,
1311 json.loads( links[ controller ] ) )
1312 else:
1313 currentLinksResult = main.FALSE
1314 utilities.assert_equals( expect=main.TRUE,
1315 actual=currentLinksResult,
1316 onpass="ONOS" + controllerStr +
1317 " links view is correct",
1318 onfail="ONOS" + controllerStr +
1319 " links view is incorrect" )
1320
Jon Halla440e872016-03-31 15:15:50 -07001321 if hosts[ controller ] and "Error" not in hosts[ controller ]:
Jon Hall85794ff2015-07-08 14:12:30 -07001322 currentHostsResult = main.Mininet1.compareHosts(
1323 mnHosts,
1324 hosts[ controller ] )
1325 else:
1326 currentHostsResult = main.FALSE
1327 utilities.assert_equals( expect=main.TRUE,
1328 actual=currentHostsResult,
1329 onpass="ONOS" + controllerStr +
1330 " hosts exist in Mininet",
1331 onfail="ONOS" + controllerStr +
1332 " hosts don't match Mininet" )
1333
1334 devicesResults = devicesResults and currentDevicesResult
1335 linksResults = linksResults and currentLinksResult
1336 hostsResults = hostsResults and currentHostsResult
1337
1338 main.step( "Device information is correct" )
1339 utilities.assert_equals(
1340 expect=main.TRUE,
1341 actual=devicesResults,
1342 onpass="Device information is correct",
1343 onfail="Device information is incorrect" )
1344
1345 main.step( "Links are correct" )
1346 utilities.assert_equals(
1347 expect=main.TRUE,
1348 actual=linksResults,
1349 onpass="Link are correct",
1350 onfail="Links are incorrect" )
1351
1352 main.step( "Hosts are correct" )
1353 utilities.assert_equals(
1354 expect=main.TRUE,
1355 actual=hostsResults,
1356 onpass="Hosts are correct",
1357 onfail="Hosts are incorrect" )
1358
1359 def CASE6( self, main ):
1360 """
1361 The Failure case.
1362 """
1363 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001364 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001365 assert main, "main not defined"
1366 assert utilities.assert_equals, "utilities.assert_equals not defined"
1367
1368 # Reset non-persistent variables
1369 try:
1370 iCounterValue = 0
1371 except NameError:
1372 main.log.error( "iCounterValue not defined, setting to 0" )
1373 iCounterValue = 0
1374
1375 main.case( "Restart ONOS node" )
Jon Hall783bbf92015-07-23 14:33:19 -07001376 main.caseExplanation = "Killing ONOS process and restart cli " +\
Jon Hall85794ff2015-07-08 14:12:30 -07001377 "sessions once onos is up."
Jon Hall96091e62015-09-21 17:34:17 -07001378
1379 main.step( "Checking ONOS Logs for errors" )
1380 for node in main.nodes:
1381 main.log.debug( "Checking logs for errors on " + node.name + ":" )
1382 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
1383
Jon Hall85794ff2015-07-08 14:12:30 -07001384 main.step( "Killing ONOS processes" )
Jon Halle1a3b752015-07-22 13:02:46 -07001385 killResult = main.ONOSbench.onosKill( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001386 start = time.time()
1387 utilities.assert_equals( expect=main.TRUE, actual=killResult,
1388 onpass="ONOS Killed",
1389 onfail="Error killing ONOS" )
1390
1391 main.step( "Checking if ONOS is up yet" )
1392 count = 0
1393 while count < 10:
Jon Halle1a3b752015-07-22 13:02:46 -07001394 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001395 if onos1Isup == main.TRUE:
1396 elapsed = time.time() - start
1397 break
1398 else:
1399 count = count + 1
1400 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
1401 onpass="ONOS is back up",
1402 onfail="ONOS failed to start" )
1403
Jon Hall6509dbf2016-06-21 17:01:17 -07001404 main.step( "Starting ONOS CLI sessions" )
Jon Halle1a3b752015-07-22 13:02:46 -07001405 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
Jon Hall85794ff2015-07-08 14:12:30 -07001406 utilities.assert_equals( expect=main.TRUE, actual=cliResults,
1407 onpass="ONOS cli startup successful",
1408 onfail="ONOS cli startup failed" )
1409
1410 if elapsed:
1411 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
1412 str( elapsed ) )
1413 main.restartTime = elapsed
1414 else:
1415 main.restartTime = -1
1416 time.sleep( 5 )
1417 # rerun on election apps
1418 main.ONOScli1.electionTestRun()
1419
1420 def CASE7( self, main ):
1421 """
1422 Check state after ONOS failure
1423 """
1424 import json
Jon Halle1a3b752015-07-22 13:02:46 -07001425 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001426 assert main, "main not defined"
1427 assert utilities.assert_equals, "utilities.assert_equals not defined"
1428 main.case( "Running ONOS Constant State Tests" )
Jon Hall6e709752016-02-01 13:38:46 -08001429
Jon Hall85794ff2015-07-08 14:12:30 -07001430 main.step( "Check that each switch has a master" )
1431 # Assert that each device has a master
Jon Halla440e872016-03-31 15:15:50 -07001432 rolesNotNull = main.TRUE
1433 threads = []
1434 for i in main.activeNodes:
1435 t = main.Thread( target=main.CLIs[i].rolesNotNull,
1436 name="rolesNotNull-" + str( i ),
1437 args=[ ] )
1438 threads.append( t )
1439 t.start()
1440
1441 for t in threads:
1442 t.join()
1443 rolesNotNull = rolesNotNull and t.result
Jon Hall85794ff2015-07-08 14:12:30 -07001444 utilities.assert_equals(
1445 expect=main.TRUE,
1446 actual=rolesNotNull,
1447 onpass="Each device has a master",
1448 onfail="Some devices don't have a master assigned" )
1449
1450 main.step( "Check if switch roles are consistent across all nodes" )
1451 ONOS1Mastership = main.ONOScli1.roles()
1452 # FIXME: Refactor this whole case for single instance
1453 if "Error" in ONOS1Mastership or not ONOS1Mastership:
1454 main.log.error( "Error in getting ONOS mastership" )
1455 main.log.warn( "ONOS1 mastership response: " +
1456 repr( ONOS1Mastership ) )
1457 consistentMastership = main.FALSE
1458 else:
1459 consistentMastership = main.TRUE
1460 utilities.assert_equals(
1461 expect=main.TRUE,
1462 actual=consistentMastership,
1463 onpass="Switch roles are consistent across all ONOS nodes",
1464 onfail="ONOS nodes have different views of switch roles" )
1465
1466 description2 = "Compare switch roles from before failure"
1467 main.step( description2 )
1468
1469 currentJson = json.loads( ONOS1Mastership )
1470 oldJson = json.loads( mastershipState )
1471 mastershipCheck = main.TRUE
1472 for i in range( 1, 29 ):
1473 switchDPID = str(
1474 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
1475
1476 current = [ switch[ 'master' ] for switch in currentJson
1477 if switchDPID in switch[ 'id' ] ]
1478 old = [ switch[ 'master' ] for switch in oldJson
1479 if switchDPID in switch[ 'id' ] ]
1480 if current == old:
1481 mastershipCheck = mastershipCheck and main.TRUE
1482 else:
1483 main.log.warn( "Mastership of switch %s changed" % switchDPID )
1484 mastershipCheck = main.FALSE
1485 utilities.assert_equals(
1486 expect=main.TRUE,
1487 actual=mastershipCheck,
1488 onpass="Mastership of Switches was not changed",
1489 onfail="Mastership of some switches changed" )
1490 mastershipCheck = mastershipCheck and consistentMastership
1491
1492 main.step( "Get the intents and compare across all nodes" )
1493 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1494 intentCheck = main.FALSE
1495 if "Error" in ONOS1Intents or not ONOS1Intents:
1496 main.log.error( "Error in getting ONOS intents" )
1497 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1498 else:
1499 intentCheck = main.TRUE
1500 utilities.assert_equals(
1501 expect=main.TRUE,
1502 actual=intentCheck,
1503 onpass="Intents are consistent across all ONOS nodes",
1504 onfail="ONOS nodes have different views of intents" )
1505 # Print the intent states
1506 intents = []
1507 intents.append( ONOS1Intents )
1508 intentStates = []
1509 for node in intents: # Iter through ONOS nodes
1510 nodeStates = []
1511 # Iter through intents of a node
1512 for intent in json.loads( node ):
1513 nodeStates.append( intent[ 'state' ] )
1514 intentStates.append( nodeStates )
1515 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1516 main.log.info( dict( out ) )
1517
1518 # NOTE: Store has no durability, so intents are lost across system
1519 # restarts
1520 """
1521 main.step( "Compare current intents with intents before the failure" )
1522 # NOTE: this requires case 5 to pass for intentState to be set.
1523 # maybe we should stop the test if that fails?
1524 sameIntents = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07001525 try:
1526 intentState
1527 except NameError:
1528 main.log.warn( "No previous intent state was saved" )
1529 else:
1530 if intentState and intentState == ONOSIntents[ 0 ]:
1531 sameIntents = main.TRUE
1532 main.log.info( "Intents are consistent with before failure" )
1533 # TODO: possibly the states have changed? we may need to figure out
1534 # what the acceptable states are
1535 elif len( intentState ) == len( ONOSIntents[ 0 ] ):
1536 sameIntents = main.TRUE
1537 try:
1538 before = json.loads( intentState )
1539 after = json.loads( ONOSIntents[ 0 ] )
1540 for intent in before:
1541 if intent not in after:
1542 sameIntents = main.FALSE
1543 main.log.debug( "Intent is not currently in ONOS " +
1544 "(at least in the same form):" )
1545 main.log.debug( json.dumps( intent ) )
1546 except ( ValueError, TypeError ):
1547 main.log.exception( "Exception printing intents" )
1548 main.log.debug( repr( ONOSIntents[0] ) )
1549 main.log.debug( repr( intentState ) )
1550 if sameIntents == main.FALSE:
1551 try:
1552 main.log.debug( "ONOS intents before: " )
1553 main.log.debug( json.dumps( json.loads( intentState ),
1554 sort_keys=True, indent=4,
1555 separators=( ',', ': ' ) ) )
1556 main.log.debug( "Current ONOS intents: " )
1557 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
1558 sort_keys=True, indent=4,
1559 separators=( ',', ': ' ) ) )
1560 except ( ValueError, TypeError ):
1561 main.log.exception( "Exception printing intents" )
1562 main.log.debug( repr( ONOSIntents[0] ) )
1563 main.log.debug( repr( intentState ) )
1564 utilities.assert_equals(
1565 expect=main.TRUE,
1566 actual=sameIntents,
1567 onpass="Intents are consistent with before failure",
1568 onfail="The Intents changed during failure" )
Jon Hall85794ff2015-07-08 14:12:30 -07001569 intentCheck = intentCheck and sameIntents
1570 """
1571 main.step( "Get the OF Table entries and compare to before " +
1572 "component failure" )
1573 FlowTables = main.TRUE
Jon Hall85794ff2015-07-08 14:12:30 -07001574 for i in range( 28 ):
1575 main.log.info( "Checking flow table on s" + str( i + 1 ) )
GlennRC68467eb2015-11-16 18:01:01 -08001576 tmpFlows = main.Mininet1.getFlowTable( "s" + str( i + 1 ), version="1.3", debug=False )
Jon Hall41d39f12016-04-11 22:54:35 -07001577 curSwitch = main.Mininet1.flowTableComp( flows[i], tmpFlows )
1578 FlowTables = FlowTables and curSwitch
1579 if curSwitch == main.FALSE:
GlennRC68467eb2015-11-16 18:01:01 -08001580 main.log.warn( "Differences in flow table for switch: s{}".format( i + 1 ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001581 utilities.assert_equals(
1582 expect=main.TRUE,
1583 actual=FlowTables,
1584 onpass="No changes were found in the flow tables",
1585 onfail="Changes were found in the flow tables" )
1586
1587 main.step( "Leadership Election is still functional" )
1588 # Test of LeadershipElection
1589
Jon Halla440e872016-03-31 15:15:50 -07001590 leader = main.nodes[ main.activeNodes[ 0 ] ].ip_address
Jon Hall85794ff2015-07-08 14:12:30 -07001591 leaderResult = main.TRUE
Jon Halle1a3b752015-07-22 13:02:46 -07001592 for controller in range( 1, main.numCtrls + 1 ):
Jon Hall85794ff2015-07-08 14:12:30 -07001593 # loop through ONOScli handlers
1594 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
1595 leaderN = node.electionTestLeader()
1596 # verify leader is ONOS1
1597 # NOTE even though we restarted ONOS, it is the only one so onos 1
1598 # must be leader
1599 if leaderN == leader:
1600 # all is well
1601 pass
1602 elif leaderN == main.FALSE:
1603 # error in response
1604 main.log.error( "Something is wrong with " +
1605 "electionTestLeader function, check the" +
1606 " error logs" )
1607 leaderResult = main.FALSE
1608 elif leader != leaderN:
1609 leaderResult = main.FALSE
1610 main.log.error( "ONOS" + str( controller ) + " sees " +
1611 str( leaderN ) +
1612 " as the leader of the election app. " +
1613 "Leader should be " + str( leader ) )
1614 utilities.assert_equals(
1615 expect=main.TRUE,
1616 actual=leaderResult,
1617 onpass="Leadership election passed",
1618 onfail="Something went wrong with Leadership election" )
1619
1620 def CASE8( self, main ):
1621 """
1622 Compare topo
1623 """
1624 import json
1625 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001626 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001627 assert main, "main not defined"
1628 assert utilities.assert_equals, "utilities.assert_equals not defined"
1629
1630 main.case( "Compare ONOS Topology view to Mininet topology" )
Jon Hall783bbf92015-07-23 14:33:19 -07001631 main.caseExplanation = "Compare topology objects between Mininet" +\
Jon Hall85794ff2015-07-08 14:12:30 -07001632 " and ONOS"
Jon Hall85794ff2015-07-08 14:12:30 -07001633 topoResult = main.FALSE
1634 elapsed = 0
1635 count = 0
Jon Halle9b1fa32015-12-08 15:32:21 -08001636 main.step( "Comparing ONOS topology to MN topology" )
Jon Hall85794ff2015-07-08 14:12:30 -07001637 startTime = time.time()
1638 # Give time for Gossip to work
Jon Halle9b1fa32015-12-08 15:32:21 -08001639 while topoResult == main.FALSE and ( elapsed < 60 or count < 3 ):
Jon Hall96091e62015-09-21 17:34:17 -07001640 devicesResults = main.TRUE
1641 linksResults = main.TRUE
1642 hostsResults = main.TRUE
1643 hostAttachmentResults = True
Jon Hall85794ff2015-07-08 14:12:30 -07001644 count += 1
1645 cliStart = time.time()
1646 devices = []
1647 devices.append( main.ONOScli1.devices() )
1648 hosts = []
1649 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1650 ipResult = main.TRUE
1651 for controller in range( 0, len( hosts ) ):
1652 controllerStr = str( controller + 1 )
1653 for host in hosts[ controller ]:
1654 if host is None or host.get( 'ipAddresses', [] ) == []:
1655 main.log.error(
1656 "DEBUG:Error with host ips on controller" +
1657 controllerStr + ": " + str( host ) )
1658 ipResult = main.FALSE
1659 ports = []
1660 ports.append( main.ONOScli1.ports() )
1661 links = []
1662 links.append( main.ONOScli1.links() )
1663 clusters = []
1664 clusters.append( main.ONOScli1.clusters() )
1665
1666 elapsed = time.time() - startTime
1667 cliTime = time.time() - cliStart
1668 print "CLI time: " + str( cliTime )
1669
1670 mnSwitches = main.Mininet1.getSwitches()
1671 mnLinks = main.Mininet1.getLinks()
1672 mnHosts = main.Mininet1.getHosts()
Jon Halle1a3b752015-07-22 13:02:46 -07001673 for controller in range( main.numCtrls ):
Jon Hall85794ff2015-07-08 14:12:30 -07001674 controllerStr = str( controller + 1 )
1675 if devices[ controller ] and ports[ controller ] and\
1676 "Error" not in devices[ controller ] and\
1677 "Error" not in ports[ controller ]:
1678
Jon Hallc6793552016-01-19 14:18:37 -08001679 try:
1680 currentDevicesResult = main.Mininet1.compareSwitches(
1681 mnSwitches,
1682 json.loads( devices[ controller ] ),
1683 json.loads( ports[ controller ] ) )
1684 except ( TypeError, ValueError ) as e:
1685 main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
1686 devices[ controller ], ports[ controller ] ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001687 else:
1688 currentDevicesResult = main.FALSE
1689 utilities.assert_equals( expect=main.TRUE,
1690 actual=currentDevicesResult,
1691 onpass="ONOS" + controllerStr +
1692 " Switches view is correct",
1693 onfail="ONOS" + controllerStr +
1694 " Switches view is incorrect" )
1695
1696 if links[ controller ] and "Error" not in links[ controller ]:
1697 currentLinksResult = main.Mininet1.compareLinks(
1698 mnSwitches, mnLinks,
1699 json.loads( links[ controller ] ) )
1700 else:
1701 currentLinksResult = main.FALSE
1702 utilities.assert_equals( expect=main.TRUE,
1703 actual=currentLinksResult,
1704 onpass="ONOS" + controllerStr +
1705 " links view is correct",
1706 onfail="ONOS" + controllerStr +
1707 " links view is incorrect" )
1708
1709 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1710 currentHostsResult = main.Mininet1.compareHosts(
1711 mnHosts,
1712 hosts[ controller ] )
1713 else:
1714 currentHostsResult = main.FALSE
1715 utilities.assert_equals( expect=main.TRUE,
1716 actual=currentHostsResult,
1717 onpass="ONOS" + controllerStr +
1718 " hosts exist in Mininet",
1719 onfail="ONOS" + controllerStr +
1720 " hosts don't match Mininet" )
1721 # CHECKING HOST ATTACHMENT POINTS
1722 hostAttachment = True
1723 zeroHosts = False
1724 # FIXME: topo-HA/obelisk specific mappings:
1725 # key is mac and value is dpid
1726 mappings = {}
1727 for i in range( 1, 29 ): # hosts 1 through 28
1728 # set up correct variables:
1729 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2)
1730 if i == 1:
1731 deviceId = "1000".zfill(16)
1732 elif i == 2:
1733 deviceId = "2000".zfill(16)
1734 elif i == 3:
1735 deviceId = "3000".zfill(16)
1736 elif i == 4:
1737 deviceId = "3004".zfill(16)
1738 elif i == 5:
1739 deviceId = "5000".zfill(16)
1740 elif i == 6:
1741 deviceId = "6000".zfill(16)
1742 elif i == 7:
1743 deviceId = "6007".zfill(16)
1744 elif i >= 8 and i <= 17:
1745 dpid = '3' + str( i ).zfill( 3 )
1746 deviceId = dpid.zfill(16)
1747 elif i >= 18 and i <= 27:
1748 dpid = '6' + str( i ).zfill( 3 )
1749 deviceId = dpid.zfill(16)
1750 elif i == 28:
1751 deviceId = "2800".zfill(16)
1752 mappings[ macId ] = deviceId
1753 if hosts[ controller ] or "Error" not in hosts[ controller ]:
1754 if hosts[ controller ] == []:
1755 main.log.warn( "There are no hosts discovered" )
1756 zeroHosts = True
1757 else:
1758 for host in hosts[ controller ]:
1759 mac = None
1760 location = None
1761 device = None
1762 port = None
1763 try:
1764 mac = host.get( 'mac' )
1765 assert mac, "mac field could not be found for this host object"
1766
1767 location = host.get( 'location' )
1768 assert location, "location field could not be found for this host object"
1769
1770 # Trim the protocol identifier off deviceId
1771 device = str( location.get( 'elementId' ) ).split(':')[1]
1772 assert device, "elementId field could not be found for this host location object"
1773
1774 port = location.get( 'port' )
1775 assert port, "port field could not be found for this host location object"
1776
1777 # Now check if this matches where they should be
1778 if mac and device and port:
1779 if str( port ) != "1":
1780 main.log.error( "The attachment port is incorrect for " +
1781 "host " + str( mac ) +
1782 ". Expected: 1 Actual: " + str( port) )
1783 hostAttachment = False
1784 if device != mappings[ str( mac ) ]:
1785 main.log.error( "The attachment device is incorrect for " +
1786 "host " + str( mac ) +
1787 ". Expected: " + mappings[ str( mac ) ] +
1788 " Actual: " + device )
1789 hostAttachment = False
1790 else:
1791 hostAttachment = False
1792 except AssertionError:
1793 main.log.exception( "Json object not as expected" )
1794 main.log.error( repr( host ) )
1795 hostAttachment = False
1796 else:
1797 main.log.error( "No hosts json output or \"Error\"" +
1798 " in output. hosts = " +
1799 repr( hosts[ controller ] ) )
1800 if zeroHosts is False:
1801 hostAttachment = True
1802
Jon Hall85794ff2015-07-08 14:12:30 -07001803 devicesResults = devicesResults and currentDevicesResult
1804 linksResults = linksResults and currentLinksResult
1805 hostsResults = hostsResults and currentHostsResult
1806 hostAttachmentResults = hostAttachmentResults and\
1807 hostAttachment
1808
Jon Halla440e872016-03-31 15:15:50 -07001809 # "consistent" results don't make sense for single instance
Jon Hall85794ff2015-07-08 14:12:30 -07001810 # there should always only be one cluster
Jon Hall85794ff2015-07-08 14:12:30 -07001811 clusterResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07001812 try:
1813 numClusters = len( json.loads( clusters[ 0 ] ) )
1814 except ( ValueError, TypeError ):
1815 main.log.exception( "Error parsing clusters[0]: " +
1816 repr( clusters[0] ) )
1817 numClusters = "ERROR"
1818 clusterResults = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07001819 if numClusters == 1:
1820 clusterResults = main.TRUE
1821 utilities.assert_equals(
1822 expect=1,
1823 actual=numClusters,
1824 onpass="ONOS shows 1 SCC",
1825 onfail="ONOS shows " + str( numClusters ) + " SCCs" )
1826
1827 topoResult = ( devicesResults and linksResults
1828 and hostsResults and ipResult and clusterResults and
1829 hostAttachmentResults )
1830
1831 topoResult = topoResult and int( count <= 2 )
1832 note = "note it takes about " + str( int( cliTime ) ) + \
1833 " seconds for the test to make all the cli calls to fetch " +\
1834 "the topology from each ONOS instance"
1835 main.log.info(
1836 "Very crass estimate for topology discovery/convergence( " +
1837 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1838 str( count ) + " tries" )
1839 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1840 onpass="Topology Check Test successful",
1841 onfail="Topology Check Test NOT successful" )
Jon Hall41d39f12016-04-11 22:54:35 -07001842 main.step( "Checking ONOS nodes" )
1843 nodeResults = utilities.retry( main.HA.nodesCheck,
1844 False,
1845 args=[main.activeNodes],
1846 attempts=5 )
1847
1848 utilities.assert_equals( expect=True, actual=nodeResults,
1849 onpass="Nodes check successful",
1850 onfail="Nodes check NOT successful" )
1851 if not nodeResults:
1852 for i in main.activeNodes:
1853 main.log.debug( "{} components not ACTIVE: \n{}".format(
1854 main.CLIs[i].name,
1855 main.CLIs[i].sendline( "scr:list | grep -v ACTIVE" ) ) )
Jon Hall85794ff2015-07-08 14:12:30 -07001856
Jon Halld2871c22016-07-26 11:01:14 -07001857 if not topoResult:
1858 main.cleanup()
1859 main.exit()
1860
Jon Hall85794ff2015-07-08 14:12:30 -07001861 def CASE9( self, main ):
1862 """
1863 Link s3-s28 down
1864 """
1865 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001866 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001867 assert main, "main not defined"
1868 assert utilities.assert_equals, "utilities.assert_equals not defined"
1869 # NOTE: You should probably run a topology check after this
1870
1871 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1872
1873 description = "Turn off a link to ensure that Link Discovery " +\
1874 "is working properly"
1875 main.case( description )
1876
1877 main.step( "Kill Link between s3 and s28" )
1878 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
1879 main.log.info( "Waiting " + str( linkSleep ) +
1880 " seconds for link down to be discovered" )
1881 time.sleep( linkSleep )
1882 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
1883 onpass="Link down successful",
1884 onfail="Failed to bring link down" )
1885 # TODO do some sort of check here
1886
1887 def CASE10( self, main ):
1888 """
1889 Link s3-s28 up
1890 """
1891 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001892 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001893 assert main, "main not defined"
1894 assert utilities.assert_equals, "utilities.assert_equals not defined"
1895 # NOTE: You should probably run a topology check after this
1896
1897 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
1898
1899 description = "Restore a link to ensure that Link Discovery is " + \
1900 "working properly"
1901 main.case( description )
1902
1903 main.step( "Bring link between s3 and s28 back up" )
1904 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
1905 main.log.info( "Waiting " + str( linkSleep ) +
1906 " seconds for link up to be discovered" )
1907 time.sleep( linkSleep )
1908 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
1909 onpass="Link up successful",
1910 onfail="Failed to bring link up" )
1911 # TODO do some sort of check here
1912
1913 def CASE11( self, main ):
1914 """
1915 Switch Down
1916 """
1917 # NOTE: You should probably run a topology check after this
1918 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001919 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001920 assert main, "main not defined"
1921 assert utilities.assert_equals, "utilities.assert_equals not defined"
1922
1923 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1924
1925 description = "Killing a switch to ensure it is discovered correctly"
Jon Halla440e872016-03-31 15:15:50 -07001926 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall85794ff2015-07-08 14:12:30 -07001927 main.case( description )
1928 switch = main.params[ 'kill' ][ 'switch' ]
1929 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1930
1931 # TODO: Make this switch parameterizable
1932 main.step( "Kill " + switch )
1933 main.log.info( "Deleting " + switch )
1934 main.Mininet1.delSwitch( switch )
1935 main.log.info( "Waiting " + str( switchSleep ) +
1936 " seconds for switch down to be discovered" )
1937 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07001938 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall85794ff2015-07-08 14:12:30 -07001939 # Peek at the deleted switch
1940 main.log.warn( str( device ) )
1941 result = main.FALSE
1942 if device and device[ 'available' ] is False:
1943 result = main.TRUE
1944 utilities.assert_equals( expect=main.TRUE, actual=result,
1945 onpass="Kill switch successful",
1946 onfail="Failed to kill switch?" )
1947
1948 def CASE12( self, main ):
1949 """
1950 Switch Up
1951 """
1952 # NOTE: You should probably run a topology check after this
1953 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001954 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001955 assert main, "main not defined"
1956 assert utilities.assert_equals, "utilities.assert_equals not defined"
1957
1958 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
1959 switch = main.params[ 'kill' ][ 'switch' ]
1960 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1961 links = main.params[ 'kill' ][ 'links' ].split()
Jon Halla440e872016-03-31 15:15:50 -07001962 onosCli = main.CLIs[ main.activeNodes[0] ]
Jon Hall85794ff2015-07-08 14:12:30 -07001963 description = "Adding a switch to ensure it is discovered correctly"
1964 main.case( description )
1965
1966 main.step( "Add back " + switch )
1967 main.Mininet1.addSwitch( switch, dpid=switchDPID )
1968 for peer in links:
1969 main.Mininet1.addLink( switch, peer )
Jon Halla440e872016-03-31 15:15:50 -07001970 ipList = [ node.ip_address for node in main.nodes ]
Jon Hall85794ff2015-07-08 14:12:30 -07001971 main.Mininet1.assignSwController( sw=switch, ip=ipList )
1972 main.log.info( "Waiting " + str( switchSleep ) +
1973 " seconds for switch up to be discovered" )
1974 time.sleep( switchSleep )
Jon Halla440e872016-03-31 15:15:50 -07001975 device = onosCli.getDevice( dpid=switchDPID )
Jon Hall85794ff2015-07-08 14:12:30 -07001976 # Peek at the deleted switch
1977 main.log.warn( str( device ) )
1978 result = main.FALSE
1979 if device and device[ 'available' ]:
1980 result = main.TRUE
1981 utilities.assert_equals( expect=main.TRUE, actual=result,
1982 onpass="add switch successful",
1983 onfail="Failed to add switch?" )
1984
1985 def CASE13( self, main ):
1986 """
1987 Clean up
1988 """
1989 import os
1990 import time
Jon Halle1a3b752015-07-22 13:02:46 -07001991 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07001992 assert main, "main not defined"
1993 assert utilities.assert_equals, "utilities.assert_equals not defined"
1994 # printing colors to terminal
1995 colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
1996 'blue': '\033[94m', 'green': '\033[92m',
1997 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
1998 main.case( "Test Cleanup" )
1999 main.step( "Killing tcpdumps" )
2000 main.Mininet2.stopTcpdump()
2001
2002 testname = main.TEST
Jon Hall96091e62015-09-21 17:34:17 -07002003 if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
Jon Hall85794ff2015-07-08 14:12:30 -07002004 main.step( "Copying MN pcap and ONOS log files to test station" )
2005 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
2006 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
Jon Hall96091e62015-09-21 17:34:17 -07002007 # NOTE: MN Pcap file is being saved to logdir.
2008 # We scp this file as MN and TestON aren't necessarily the same vm
2009
2010 # FIXME: To be replaced with a Jenkin's post script
Jon Hall85794ff2015-07-08 14:12:30 -07002011 # TODO: Load these from params
2012 # NOTE: must end in /
2013 logFolder = "/opt/onos/log/"
2014 logFiles = [ "karaf.log", "karaf.log.1" ]
2015 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07002016 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07002017 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002018 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002019 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2020 logFolder + f, dstName )
Jon Hall85794ff2015-07-08 14:12:30 -07002021 # std*.log's
2022 # NOTE: must end in /
2023 logFolder = "/opt/onos/var/"
2024 logFiles = [ "stderr.log", "stdout.log" ]
2025 # NOTE: must end in /
Jon Hall85794ff2015-07-08 14:12:30 -07002026 for f in logFiles:
Jon Hall96091e62015-09-21 17:34:17 -07002027 for node in main.nodes:
Jon Halla440e872016-03-31 15:15:50 -07002028 dstName = main.logdir + "/" + node.name + "-" + f
Jon Hall96091e62015-09-21 17:34:17 -07002029 main.ONOSbench.secureCopy( node.user_name, node.ip_address,
2030 logFolder + f, dstName )
2031 else:
2032 main.log.debug( "skipping saving log files" )
Jon Hall85794ff2015-07-08 14:12:30 -07002033
Jon Hall85794ff2015-07-08 14:12:30 -07002034 main.step( "Stopping Mininet" )
2035 mnResult = main.Mininet1.stopNet()
2036 utilities.assert_equals( expect=main.TRUE, actual=mnResult,
2037 onpass="Mininet stopped",
2038 onfail="MN cleanup NOT successful" )
2039
2040 main.step( "Checking ONOS Logs for errors" )
Jon Hall96091e62015-09-21 17:34:17 -07002041 for node in main.nodes:
2042 main.log.debug( "Checking logs for errors on " + node.name + ":" )
2043 main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
Jon Hall85794ff2015-07-08 14:12:30 -07002044
2045 try:
2046 timerLog = open( main.logdir + "/Timers.csv", 'w')
2047 # Overwrite with empty line and close
2048 labels = "Gossip Intents, Restart"
2049 data = str( gossipTime ) + ", " + str( main.restartTime )
2050 timerLog.write( labels + "\n" + data )
2051 timerLog.close()
2052 except NameError, e:
2053 main.log.exception(e)
2054
2055 def CASE14( self, main ):
2056 """
2057 start election app on all onos nodes
2058 """
Jon Halle1a3b752015-07-22 13:02:46 -07002059 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002060 assert main, "main not defined"
2061 assert utilities.assert_equals, "utilities.assert_equals not defined"
2062
2063 main.case("Start Leadership Election app")
2064 main.step( "Install leadership election app" )
Jon Halla440e872016-03-31 15:15:50 -07002065 onosCli = main.CLIs[ main.activeNodes[0] ]
2066 appResult = onosCli.activateApp( "org.onosproject.election" )
Jon Hall85794ff2015-07-08 14:12:30 -07002067 utilities.assert_equals(
2068 expect=main.TRUE,
2069 actual=appResult,
2070 onpass="Election app installed",
2071 onfail="Something went wrong with installing Leadership election" )
2072
2073 main.step( "Run for election on each node" )
Jon Halla440e872016-03-31 15:15:50 -07002074 for i in main.activeNodes:
2075 main.CLIs[i].electionTestRun()
Jon Hall25463a82016-04-13 14:03:52 -07002076 time.sleep(5)
2077 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
2078 sameResult, leaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall85794ff2015-07-08 14:12:30 -07002079 utilities.assert_equals(
Jon Hall25463a82016-04-13 14:03:52 -07002080 expect=True,
2081 actual=sameResult,
2082 onpass="All nodes see the same leaderboards",
2083 onfail="Inconsistent leaderboards" )
2084
2085 if sameResult:
2086 leader = leaders[ 0 ][ 0 ]
2087 if main.nodes[main.activeNodes[0]].ip_address in leader:
2088 correctLeader = True
2089 else:
2090 correctLeader = False
2091 main.step( "First node was elected leader" )
2092 utilities.assert_equals(
2093 expect=True,
2094 actual=correctLeader,
2095 onpass="Correct leader was elected",
2096 onfail="Incorrect leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07002097
2098 def CASE15( self, main ):
2099 """
2100 Check that Leadership Election is still functional
acsmars71adceb2015-08-31 15:09:26 -07002101 15.1 Run election on each node
2102 15.2 Check that each node has the same leaders and candidates
2103 15.3 Find current leader and withdraw
2104 15.4 Check that a new node was elected leader
2105 15.5 Check that that new leader was the candidate of old leader
2106 15.6 Run for election on old leader
2107 15.7 Check that oldLeader is a candidate, and leader if only 1 node
2108 15.8 Make sure that the old leader was added to the candidate list
2109
2110 old and new variable prefixes refer to data from before vs after
2111 withdrawl and later before withdrawl vs after re-election
Jon Hall85794ff2015-07-08 14:12:30 -07002112 """
acsmars71adceb2015-08-31 15:09:26 -07002113 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002114 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002115 assert main, "main not defined"
2116 assert utilities.assert_equals, "utilities.assert_equals not defined"
acsmars71adceb2015-08-31 15:09:26 -07002117 assert main.CLIs, "main.CLIs not defined"
2118 assert main.nodes, "main.nodes not defined"
2119
Jon Hall85794ff2015-07-08 14:12:30 -07002120 description = "Check that Leadership Election is still functional"
2121 main.case( description )
Jon Halla440e872016-03-31 15:15:50 -07002122 # NOTE: Need to re-run after restarts since being a canidate is not persistant
acsmars2c2fcdd2015-08-25 17:14:13 -07002123
Jon Halla440e872016-03-31 15:15:50 -07002124 oldLeaders = [] # list of lists of each nodes' candidates before
2125 newLeaders = [] # list of lists of each nodes' candidates after
acsmars71adceb2015-08-31 15:09:26 -07002126 oldLeader = '' # the old leader from oldLeaders, None if not same
2127 newLeader = '' # the new leaders fron newLoeaders, None if not same
2128 oldLeaderCLI = None # the CLI of the old leader used for re-electing
2129 expectNoLeader = False # True when there is only one leader
2130 if main.numCtrls == 1:
2131 expectNoLeader = True
acsmars2c2fcdd2015-08-25 17:14:13 -07002132
acsmars71adceb2015-08-31 15:09:26 -07002133 main.step( "Run for election on each node" )
2134 electionResult = main.TRUE
2135
Jon Halla440e872016-03-31 15:15:50 -07002136 for i in main.activeNodes: # run test election on each node
2137 if main.CLIs[i].electionTestRun() == main.FALSE:
acsmars71adceb2015-08-31 15:09:26 -07002138 electionResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002139 utilities.assert_equals(
2140 expect=main.TRUE,
2141 actual=electionResult,
2142 onpass="All nodes successfully ran for leadership",
2143 onfail="At least one node failed to run for leadership" )
2144
acsmars3a72bde2015-09-02 14:16:22 -07002145 if electionResult == main.FALSE:
2146 main.log.error(
2147 "Skipping Test Case because Election Test App isn't loaded" )
2148 main.skipCase()
2149
acsmars71adceb2015-08-31 15:09:26 -07002150 main.step( "Check that each node shows the same leader and candidates" )
Jon Halla440e872016-03-31 15:15:50 -07002151 failMessage = "Nodes have different leaderboards"
Jon Halla440e872016-03-31 15:15:50 -07002152 activeCLIs = [ main.CLIs[i] for i in main.activeNodes ]
Jon Hall41d39f12016-04-11 22:54:35 -07002153 sameResult, oldLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Halla440e872016-03-31 15:15:50 -07002154 if sameResult:
2155 oldLeader = oldLeaders[ 0 ][ 0 ]
2156 main.log.warn( oldLeader )
Jon Hall85794ff2015-07-08 14:12:30 -07002157 else:
Jon Halla440e872016-03-31 15:15:50 -07002158 oldLeader = None
acsmars71adceb2015-08-31 15:09:26 -07002159 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002160 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002161 actual=sameResult,
Jon Halla440e872016-03-31 15:15:50 -07002162 onpass="Leaderboards are consistent for the election topic",
acsmars71adceb2015-08-31 15:09:26 -07002163 onfail=failMessage )
2164
2165 main.step( "Find current leader and withdraw" )
2166 withdrawResult = main.TRUE
2167 # do some sanity checking on leader before using it
2168 if oldLeader is None:
2169 main.log.error( "Leadership isn't consistent." )
2170 withdrawResult = main.FALSE
2171 # Get the CLI of the oldLeader
Jon Halla440e872016-03-31 15:15:50 -07002172 for i in main.activeNodes:
acsmars71adceb2015-08-31 15:09:26 -07002173 if oldLeader == main.nodes[ i ].ip_address:
2174 oldLeaderCLI = main.CLIs[ i ]
2175 break
2176 else: # FOR/ELSE statement
2177 main.log.error( "Leader election, could not find current leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07002178 if oldLeader:
acsmars71adceb2015-08-31 15:09:26 -07002179 withdrawResult = oldLeaderCLI.electionTestWithdraw()
Jon Hall85794ff2015-07-08 14:12:30 -07002180 utilities.assert_equals(
2181 expect=main.TRUE,
2182 actual=withdrawResult,
2183 onpass="Node was withdrawn from election",
2184 onfail="Node was not withdrawn from election" )
2185
acsmars71adceb2015-08-31 15:09:26 -07002186 main.step( "Check that a new node was elected leader" )
acsmars71adceb2015-08-31 15:09:26 -07002187 failMessage = "Nodes have different leaders"
acsmars71adceb2015-08-31 15:09:26 -07002188 # Get new leaders and candidates
Jon Hall41d39f12016-04-11 22:54:35 -07002189 newLeaderResult, newLeaders = main.HA.consistentLeaderboards( activeCLIs )
Jon Hall3a7843a2016-04-12 03:01:09 -07002190 newLeader = None
Jon Halla440e872016-03-31 15:15:50 -07002191 if newLeaderResult:
Jon Hall3a7843a2016-04-12 03:01:09 -07002192 if newLeaders[ 0 ][ 0 ] == 'none':
2193 main.log.error( "No leader was elected on at least 1 node" )
2194 if not expectNoLeader:
2195 newLeaderResult = False
Jon Hall25463a82016-04-13 14:03:52 -07002196 newLeader = newLeaders[ 0 ][ 0 ]
acsmars71adceb2015-08-31 15:09:26 -07002197
2198 # Check that the new leader is not the older leader, which was withdrawn
2199 if newLeader == oldLeader:
Jon Halla440e872016-03-31 15:15:50 -07002200 newLeaderResult = False
2201 main.log.error( "All nodes still see old leader: " + str( oldLeader ) +
acsmars71adceb2015-08-31 15:09:26 -07002202 " as the current leader" )
Jon Hall85794ff2015-07-08 14:12:30 -07002203 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002204 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002205 actual=newLeaderResult,
2206 onpass="Leadership election passed",
2207 onfail="Something went wrong with Leadership election" )
Jon Hall85794ff2015-07-08 14:12:30 -07002208
Jon Halla440e872016-03-31 15:15:50 -07002209 main.step( "Check that that new leader was the candidate of old leader" )
2210 # candidates[ 2 ] should become the top candidate after withdrawl
acsmars71adceb2015-08-31 15:09:26 -07002211 correctCandidateResult = main.TRUE
2212 if expectNoLeader:
2213 if newLeader == 'none':
2214 main.log.info( "No leader expected. None found. Pass" )
2215 correctCandidateResult = main.TRUE
2216 else:
2217 main.log.info( "Expected no leader, got: " + str( newLeader ) )
2218 correctCandidateResult = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002219 elif len( oldLeaders[0] ) >= 3:
2220 if newLeader == oldLeaders[ 0 ][ 2 ]:
2221 # correct leader was elected
2222 correctCandidateResult = main.TRUE
2223 else:
2224 correctCandidateResult = main.FALSE
2225 main.log.error( "Candidate {} was elected. {} should have had priority.".format(
2226 newLeader, oldLeaders[ 0 ][ 2 ] ) )
2227 else:
2228 main.log.warn( "Could not determine who should be the correct leader" )
2229 main.log.debug( oldLeaders[ 0 ] )
acsmars71adceb2015-08-31 15:09:26 -07002230 correctCandidateResult = main.FALSE
acsmars71adceb2015-08-31 15:09:26 -07002231 utilities.assert_equals(
2232 expect=main.TRUE,
2233 actual=correctCandidateResult,
2234 onpass="Correct Candidate Elected",
2235 onfail="Incorrect Candidate Elected" )
2236
Jon Hall85794ff2015-07-08 14:12:30 -07002237 main.step( "Run for election on old leader( just so everyone " +
2238 "is in the hat )" )
acsmars71adceb2015-08-31 15:09:26 -07002239 if oldLeaderCLI is not None:
2240 runResult = oldLeaderCLI.electionTestRun()
Jon Hall85794ff2015-07-08 14:12:30 -07002241 else:
acsmars71adceb2015-08-31 15:09:26 -07002242 main.log.error( "No old leader to re-elect" )
Jon Hall85794ff2015-07-08 14:12:30 -07002243 runResult = main.FALSE
2244 utilities.assert_equals(
2245 expect=main.TRUE,
2246 actual=runResult,
2247 onpass="App re-ran for election",
2248 onfail="App failed to run for election" )
Jon Halla440e872016-03-31 15:15:50 -07002249
acsmars71adceb2015-08-31 15:09:26 -07002250 main.step(
2251 "Check that oldLeader is a candidate, and leader if only 1 node" )
2252 # verify leader didn't just change
Jon Halla440e872016-03-31 15:15:50 -07002253 # Get new leaders and candidates
2254 reRunLeaders = []
2255 time.sleep( 5 ) # Paremterize
Jon Hall41d39f12016-04-11 22:54:35 -07002256 positionResult, reRunLeaders = main.HA.consistentLeaderboards( activeCLIs )
acsmars71adceb2015-08-31 15:09:26 -07002257
2258 # Check that the re-elected node is last on the candidate List
Jon Hall3a7843a2016-04-12 03:01:09 -07002259 if not reRunLeaders[0]:
2260 positionResult = main.FALSE
2261 elif oldLeader != reRunLeaders[ 0 ][ -1 ]:
Jon Halla440e872016-03-31 15:15:50 -07002262 main.log.error( "Old Leader ({}) not in the proper position: {} ".format( str( oldLeader),
2263 str( reRunLeaders[ 0 ] ) ) )
acsmars71adceb2015-08-31 15:09:26 -07002264 positionResult = main.FALSE
Jon Hall85794ff2015-07-08 14:12:30 -07002265 utilities.assert_equals(
Jon Halla440e872016-03-31 15:15:50 -07002266 expect=True,
acsmars71adceb2015-08-31 15:09:26 -07002267 actual=positionResult,
Jon Hall85794ff2015-07-08 14:12:30 -07002268 onpass="Old leader successfully re-ran for election",
2269 onfail="Something went wrong with Leadership election after " +
2270 "the old leader re-ran for election" )
2271
2272 def CASE16( self, main ):
2273 """
2274 Install Distributed Primitives app
2275 """
Jon Halla440e872016-03-31 15:15:50 -07002276 import time
Jon Halle1a3b752015-07-22 13:02:46 -07002277 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002278 assert main, "main not defined"
2279 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002280 assert main.CLIs, "main.CLIs not defined"
2281 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002282
2283 # Variables for the distributed primitives tests
2284 global pCounterName
Jon Hall85794ff2015-07-08 14:12:30 -07002285 global pCounterValue
Jon Hall85794ff2015-07-08 14:12:30 -07002286 global onosSet
2287 global onosSetName
2288 pCounterName = "TestON-Partitions"
Jon Hall85794ff2015-07-08 14:12:30 -07002289 pCounterValue = 0
Jon Hall85794ff2015-07-08 14:12:30 -07002290 onosSet = set([])
2291 onosSetName = "TestON-set"
2292
2293 description = "Install Primitives app"
2294 main.case( description )
2295 main.step( "Install Primitives app" )
2296 appName = "org.onosproject.distributedprimitives"
Jon Halla440e872016-03-31 15:15:50 -07002297 node = main.activeNodes[0]
2298 appResults = main.CLIs[node].activateApp( appName )
Jon Hall85794ff2015-07-08 14:12:30 -07002299 utilities.assert_equals( expect=main.TRUE,
2300 actual=appResults,
2301 onpass="Primitives app activated",
2302 onfail="Primitives app not activated" )
Jon Halla440e872016-03-31 15:15:50 -07002303 time.sleep( 5 ) # To allow all nodes to activate
Jon Hall85794ff2015-07-08 14:12:30 -07002304
2305 def CASE17( self, main ):
2306 """
2307 Check for basic functionality with distributed primitives
2308 """
Jon Hall85794ff2015-07-08 14:12:30 -07002309 # Make sure variables are defined/set
Jon Halle1a3b752015-07-22 13:02:46 -07002310 assert main.numCtrls, "main.numCtrls not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002311 assert main, "main not defined"
2312 assert utilities.assert_equals, "utilities.assert_equals not defined"
Jon Halle1a3b752015-07-22 13:02:46 -07002313 assert main.CLIs, "main.CLIs not defined"
2314 assert main.nodes, "main.nodes not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002315 assert pCounterName, "pCounterName not defined"
Jon Hall85794ff2015-07-08 14:12:30 -07002316 assert onosSetName, "onosSetName not defined"
2317 # NOTE: assert fails if value is 0/None/Empty/False
2318 try:
2319 pCounterValue
2320 except NameError:
2321 main.log.error( "pCounterValue not defined, setting to 0" )
2322 pCounterValue = 0
2323 try:
Jon Hall85794ff2015-07-08 14:12:30 -07002324 onosSet
2325 except NameError:
2326 main.log.error( "onosSet not defined, setting to empty Set" )
2327 onosSet = set([])
2328 # Variables for the distributed primitives tests. These are local only
2329 addValue = "a"
2330 addAllValue = "a b c d e f"
2331 retainValue = "c d e f"
2332
2333 description = "Check for basic functionality with distributed " +\
2334 "primitives"
2335 main.case( description )
Jon Halle1a3b752015-07-22 13:02:46 -07002336 main.caseExplanation = "Test the methods of the distributed " +\
2337 "primitives (counters and sets) throught the cli"
Jon Hall85794ff2015-07-08 14:12:30 -07002338 # DISTRIBUTED ATOMIC COUNTERS
Jon Halle1a3b752015-07-22 13:02:46 -07002339 # Partitioned counters
2340 main.step( "Increment then get a default counter on each node" )
Jon Hall85794ff2015-07-08 14:12:30 -07002341 pCounters = []
2342 threads = []
2343 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002344 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002345 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2346 name="counterAddAndGet-" + str( i ),
Jon Hall85794ff2015-07-08 14:12:30 -07002347 args=[ pCounterName ] )
2348 pCounterValue += 1
2349 addedPValues.append( pCounterValue )
2350 threads.append( t )
2351 t.start()
2352
2353 for t in threads:
2354 t.join()
2355 pCounters.append( t.result )
2356 # Check that counter incremented numController times
2357 pCounterResults = True
2358 for i in addedPValues:
2359 tmpResult = i in pCounters
2360 pCounterResults = pCounterResults and tmpResult
2361 if not tmpResult:
2362 main.log.error( str( i ) + " is not in partitioned "
2363 "counter incremented results" )
2364 utilities.assert_equals( expect=True,
2365 actual=pCounterResults,
2366 onpass="Default counter incremented",
2367 onfail="Error incrementing default" +
2368 " counter" )
2369
Jon Halle1a3b752015-07-22 13:02:46 -07002370 main.step( "Get then Increment a default counter on each node" )
2371 pCounters = []
2372 threads = []
2373 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002374 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002375 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2376 name="counterGetAndAdd-" + str( i ),
2377 args=[ pCounterName ] )
2378 addedPValues.append( pCounterValue )
2379 pCounterValue += 1
2380 threads.append( t )
2381 t.start()
2382
2383 for t in threads:
2384 t.join()
2385 pCounters.append( t.result )
2386 # Check that counter incremented numController times
2387 pCounterResults = True
2388 for i in addedPValues:
2389 tmpResult = i in pCounters
2390 pCounterResults = pCounterResults and tmpResult
2391 if not tmpResult:
2392 main.log.error( str( i ) + " is not in partitioned "
2393 "counter incremented results" )
2394 utilities.assert_equals( expect=True,
2395 actual=pCounterResults,
2396 onpass="Default counter incremented",
2397 onfail="Error incrementing default" +
2398 " counter" )
2399
2400 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07002401 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07002402 utilities.assert_equals( expect=main.TRUE,
2403 actual=incrementCheck,
2404 onpass="Added counters are correct",
2405 onfail="Added counters are incorrect" )
2406
2407 main.step( "Add -8 to then get a default counter on each node" )
2408 pCounters = []
2409 threads = []
2410 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002411 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002412 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2413 name="counterIncrement-" + str( i ),
2414 args=[ pCounterName ],
2415 kwargs={ "delta": -8 } )
2416 pCounterValue += -8
2417 addedPValues.append( pCounterValue )
2418 threads.append( t )
2419 t.start()
2420
2421 for t in threads:
2422 t.join()
2423 pCounters.append( t.result )
2424 # Check that counter incremented numController times
2425 pCounterResults = True
2426 for i in addedPValues:
2427 tmpResult = i in pCounters
2428 pCounterResults = pCounterResults and tmpResult
2429 if not tmpResult:
2430 main.log.error( str( i ) + " is not in partitioned "
2431 "counter incremented results" )
2432 utilities.assert_equals( expect=True,
2433 actual=pCounterResults,
2434 onpass="Default counter incremented",
2435 onfail="Error incrementing default" +
2436 " counter" )
2437
2438 main.step( "Add 5 to then get a default counter on each node" )
2439 pCounters = []
2440 threads = []
2441 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002442 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002443 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet,
2444 name="counterIncrement-" + str( i ),
2445 args=[ pCounterName ],
2446 kwargs={ "delta": 5 } )
2447 pCounterValue += 5
2448 addedPValues.append( pCounterValue )
2449 threads.append( t )
2450 t.start()
2451
2452 for t in threads:
2453 t.join()
2454 pCounters.append( t.result )
2455 # Check that counter incremented numController times
2456 pCounterResults = True
2457 for i in addedPValues:
2458 tmpResult = i in pCounters
2459 pCounterResults = pCounterResults and tmpResult
2460 if not tmpResult:
2461 main.log.error( str( i ) + " is not in partitioned "
2462 "counter incremented results" )
2463 utilities.assert_equals( expect=True,
2464 actual=pCounterResults,
2465 onpass="Default counter incremented",
2466 onfail="Error incrementing default" +
2467 " counter" )
2468
2469 main.step( "Get then add 5 to a default counter on each node" )
2470 pCounters = []
2471 threads = []
2472 addedPValues = []
Jon Halla440e872016-03-31 15:15:50 -07002473 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002474 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd,
2475 name="counterIncrement-" + str( i ),
2476 args=[ pCounterName ],
2477 kwargs={ "delta": 5 } )
2478 addedPValues.append( pCounterValue )
2479 pCounterValue += 5
2480 threads.append( t )
2481 t.start()
2482
2483 for t in threads:
2484 t.join()
2485 pCounters.append( t.result )
2486 # Check that counter incremented numController times
2487 pCounterResults = True
2488 for i in addedPValues:
2489 tmpResult = i in pCounters
2490 pCounterResults = pCounterResults and tmpResult
2491 if not tmpResult:
2492 main.log.error( str( i ) + " is not in partitioned "
2493 "counter incremented results" )
2494 utilities.assert_equals( expect=True,
2495 actual=pCounterResults,
2496 onpass="Default counter incremented",
2497 onfail="Error incrementing default" +
2498 " counter" )
2499
2500 main.step( "Counters we added have the correct values" )
Jon Hall41d39f12016-04-11 22:54:35 -07002501 incrementCheck = main.HA.counterCheck( pCounterName, pCounterValue )
Jon Halle1a3b752015-07-22 13:02:46 -07002502 utilities.assert_equals( expect=main.TRUE,
2503 actual=incrementCheck,
2504 onpass="Added counters are correct",
2505 onfail="Added counters are incorrect" )
2506
Jon Hall85794ff2015-07-08 14:12:30 -07002507 # DISTRIBUTED SETS
2508 main.step( "Distributed Set get" )
2509 size = len( onosSet )
2510 getResponses = []
2511 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002512 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002513 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002514 name="setTestGet-" + str( i ),
2515 args=[ onosSetName ] )
2516 threads.append( t )
2517 t.start()
2518 for t in threads:
2519 t.join()
2520 getResponses.append( t.result )
2521
2522 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002523 for i in range( len( main.activeNodes ) ):
2524 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002525 if isinstance( getResponses[ i ], list):
2526 current = set( getResponses[ i ] )
2527 if len( current ) == len( getResponses[ i ] ):
2528 # no repeats
2529 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002530 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002531 " has incorrect view" +
2532 " of set " + onosSetName + ":\n" +
2533 str( getResponses[ i ] ) )
2534 main.log.debug( "Expected: " + str( onosSet ) )
2535 main.log.debug( "Actual: " + str( current ) )
2536 getResults = main.FALSE
2537 else:
2538 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002539 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002540 " has repeat elements in" +
2541 " set " + onosSetName + ":\n" +
2542 str( getResponses[ i ] ) )
2543 getResults = main.FALSE
2544 elif getResponses[ i ] == main.ERROR:
2545 getResults = main.FALSE
2546 utilities.assert_equals( expect=main.TRUE,
2547 actual=getResults,
2548 onpass="Set elements are correct",
2549 onfail="Set elements are incorrect" )
2550
2551 main.step( "Distributed Set size" )
2552 sizeResponses = []
2553 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002554 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002555 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002556 name="setTestSize-" + str( i ),
2557 args=[ onosSetName ] )
2558 threads.append( t )
2559 t.start()
2560 for t in threads:
2561 t.join()
2562 sizeResponses.append( t.result )
2563
2564 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002565 for i in range( len( main.activeNodes ) ):
2566 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002567 if size != sizeResponses[ i ]:
2568 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002569 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002570 " expected a size of " + str( size ) +
2571 " for set " + onosSetName +
2572 " but got " + str( sizeResponses[ i ] ) )
2573 utilities.assert_equals( expect=main.TRUE,
2574 actual=sizeResults,
2575 onpass="Set sizes are correct",
2576 onfail="Set sizes are incorrect" )
2577
2578 main.step( "Distributed Set add()" )
2579 onosSet.add( addValue )
2580 addResponses = []
2581 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002582 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002583 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002584 name="setTestAdd-" + str( i ),
2585 args=[ onosSetName, addValue ] )
2586 threads.append( t )
2587 t.start()
2588 for t in threads:
2589 t.join()
2590 addResponses.append( t.result )
2591
2592 # main.TRUE = successfully changed the set
2593 # main.FALSE = action resulted in no change in set
2594 # main.ERROR - Some error in executing the function
2595 addResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002596 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002597 if addResponses[ i ] == main.TRUE:
2598 # All is well
2599 pass
2600 elif addResponses[ i ] == main.FALSE:
2601 # Already in set, probably fine
2602 pass
2603 elif addResponses[ i ] == main.ERROR:
2604 # Error in execution
2605 addResults = main.FALSE
2606 else:
2607 # unexpected result
2608 addResults = main.FALSE
2609 if addResults != main.TRUE:
2610 main.log.error( "Error executing set add" )
2611
2612 # Check if set is still correct
2613 size = len( onosSet )
2614 getResponses = []
2615 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002616 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002617 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002618 name="setTestGet-" + str( i ),
2619 args=[ onosSetName ] )
2620 threads.append( t )
2621 t.start()
2622 for t in threads:
2623 t.join()
2624 getResponses.append( t.result )
2625 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002626 for i in range( len( main.activeNodes ) ):
2627 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002628 if isinstance( getResponses[ i ], list):
2629 current = set( getResponses[ i ] )
2630 if len( current ) == len( getResponses[ i ] ):
2631 # no repeats
2632 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002633 main.log.error( "ONOS" + node + " has incorrect view" +
Jon Hall85794ff2015-07-08 14:12:30 -07002634 " of set " + onosSetName + ":\n" +
2635 str( getResponses[ i ] ) )
2636 main.log.debug( "Expected: " + str( onosSet ) )
2637 main.log.debug( "Actual: " + str( current ) )
2638 getResults = main.FALSE
2639 else:
2640 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002641 main.log.error( "ONOS" + node + " has repeat elements in" +
Jon Hall85794ff2015-07-08 14:12:30 -07002642 " set " + onosSetName + ":\n" +
2643 str( getResponses[ i ] ) )
2644 getResults = main.FALSE
2645 elif getResponses[ i ] == main.ERROR:
2646 getResults = main.FALSE
2647 sizeResponses = []
2648 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002649 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002650 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002651 name="setTestSize-" + str( i ),
2652 args=[ onosSetName ] )
2653 threads.append( t )
2654 t.start()
2655 for t in threads:
2656 t.join()
2657 sizeResponses.append( t.result )
2658 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002659 for i in range( len( main.activeNodes ) ):
2660 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002661 if size != sizeResponses[ i ]:
2662 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002663 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002664 " expected a size of " + str( size ) +
2665 " for set " + onosSetName +
2666 " but got " + str( sizeResponses[ i ] ) )
2667 addResults = addResults and getResults and sizeResults
2668 utilities.assert_equals( expect=main.TRUE,
2669 actual=addResults,
2670 onpass="Set add correct",
2671 onfail="Set add was incorrect" )
2672
2673 main.step( "Distributed Set addAll()" )
2674 onosSet.update( addAllValue.split() )
2675 addResponses = []
2676 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002677 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002678 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07002679 name="setTestAddAll-" + str( i ),
2680 args=[ onosSetName, addAllValue ] )
2681 threads.append( t )
2682 t.start()
2683 for t in threads:
2684 t.join()
2685 addResponses.append( t.result )
2686
2687 # main.TRUE = successfully changed the set
2688 # main.FALSE = action resulted in no change in set
2689 # main.ERROR - Some error in executing the function
2690 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002691 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002692 if addResponses[ i ] == main.TRUE:
2693 # All is well
2694 pass
2695 elif addResponses[ i ] == main.FALSE:
2696 # Already in set, probably fine
2697 pass
2698 elif addResponses[ i ] == main.ERROR:
2699 # Error in execution
2700 addAllResults = main.FALSE
2701 else:
2702 # unexpected result
2703 addAllResults = main.FALSE
2704 if addAllResults != main.TRUE:
2705 main.log.error( "Error executing set addAll" )
2706
2707 # Check if set is still correct
2708 size = len( onosSet )
2709 getResponses = []
2710 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002711 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002712 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002713 name="setTestGet-" + str( i ),
2714 args=[ onosSetName ] )
2715 threads.append( t )
2716 t.start()
2717 for t in threads:
2718 t.join()
2719 getResponses.append( t.result )
2720 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002721 for i in range( len( main.activeNodes ) ):
2722 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002723 if isinstance( getResponses[ i ], list):
2724 current = set( getResponses[ i ] )
2725 if len( current ) == len( getResponses[ i ] ):
2726 # no repeats
2727 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002728 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002729 " has incorrect view" +
2730 " of set " + onosSetName + ":\n" +
2731 str( getResponses[ i ] ) )
2732 main.log.debug( "Expected: " + str( onosSet ) )
2733 main.log.debug( "Actual: " + str( current ) )
2734 getResults = main.FALSE
2735 else:
2736 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002737 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002738 " has repeat elements in" +
2739 " set " + onosSetName + ":\n" +
2740 str( getResponses[ i ] ) )
2741 getResults = main.FALSE
2742 elif getResponses[ i ] == main.ERROR:
2743 getResults = main.FALSE
2744 sizeResponses = []
2745 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002746 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002747 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002748 name="setTestSize-" + str( i ),
2749 args=[ onosSetName ] )
2750 threads.append( t )
2751 t.start()
2752 for t in threads:
2753 t.join()
2754 sizeResponses.append( t.result )
2755 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002756 for i in range( len( main.activeNodes ) ):
2757 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002758 if size != sizeResponses[ i ]:
2759 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002760 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002761 " expected a size of " + str( size ) +
2762 " for set " + onosSetName +
2763 " but got " + str( sizeResponses[ i ] ) )
2764 addAllResults = addAllResults and getResults and sizeResults
2765 utilities.assert_equals( expect=main.TRUE,
2766 actual=addAllResults,
2767 onpass="Set addAll correct",
2768 onfail="Set addAll was incorrect" )
2769
2770 main.step( "Distributed Set contains()" )
2771 containsResponses = []
2772 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002773 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002774 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002775 name="setContains-" + str( i ),
2776 args=[ onosSetName ],
2777 kwargs={ "values": addValue } )
2778 threads.append( t )
2779 t.start()
2780 for t in threads:
2781 t.join()
2782 # NOTE: This is the tuple
2783 containsResponses.append( t.result )
2784
2785 containsResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002786 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002787 if containsResponses[ i ] == main.ERROR:
2788 containsResults = main.FALSE
2789 else:
2790 containsResults = containsResults and\
2791 containsResponses[ i ][ 1 ]
2792 utilities.assert_equals( expect=main.TRUE,
2793 actual=containsResults,
2794 onpass="Set contains is functional",
2795 onfail="Set contains failed" )
2796
2797 main.step( "Distributed Set containsAll()" )
2798 containsAllResponses = []
2799 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002800 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002801 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002802 name="setContainsAll-" + str( i ),
2803 args=[ onosSetName ],
2804 kwargs={ "values": addAllValue } )
2805 threads.append( t )
2806 t.start()
2807 for t in threads:
2808 t.join()
2809 # NOTE: This is the tuple
2810 containsAllResponses.append( t.result )
2811
2812 containsAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002813 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002814 if containsResponses[ i ] == main.ERROR:
2815 containsResults = main.FALSE
2816 else:
2817 containsResults = containsResults and\
2818 containsResponses[ i ][ 1 ]
2819 utilities.assert_equals( expect=main.TRUE,
2820 actual=containsAllResults,
2821 onpass="Set containsAll is functional",
2822 onfail="Set containsAll failed" )
2823
2824 main.step( "Distributed Set remove()" )
2825 onosSet.remove( addValue )
2826 removeResponses = []
2827 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002828 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002829 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002830 name="setTestRemove-" + str( i ),
2831 args=[ onosSetName, addValue ] )
2832 threads.append( t )
2833 t.start()
2834 for t in threads:
2835 t.join()
2836 removeResponses.append( t.result )
2837
2838 # main.TRUE = successfully changed the set
2839 # main.FALSE = action resulted in no change in set
2840 # main.ERROR - Some error in executing the function
2841 removeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002842 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002843 if removeResponses[ i ] == main.TRUE:
2844 # All is well
2845 pass
2846 elif removeResponses[ i ] == main.FALSE:
2847 # not in set, probably fine
2848 pass
2849 elif removeResponses[ i ] == main.ERROR:
2850 # Error in execution
2851 removeResults = main.FALSE
2852 else:
2853 # unexpected result
2854 removeResults = main.FALSE
2855 if removeResults != main.TRUE:
2856 main.log.error( "Error executing set remove" )
2857
2858 # Check if set is still correct
2859 size = len( onosSet )
2860 getResponses = []
2861 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002862 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002863 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002864 name="setTestGet-" + str( i ),
2865 args=[ onosSetName ] )
2866 threads.append( t )
2867 t.start()
2868 for t in threads:
2869 t.join()
2870 getResponses.append( t.result )
2871 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002872 for i in range( len( main.activeNodes ) ):
2873 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002874 if isinstance( getResponses[ i ], list):
2875 current = set( getResponses[ i ] )
2876 if len( current ) == len( getResponses[ i ] ):
2877 # no repeats
2878 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002879 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002880 " has incorrect view" +
2881 " of set " + onosSetName + ":\n" +
2882 str( getResponses[ i ] ) )
2883 main.log.debug( "Expected: " + str( onosSet ) )
2884 main.log.debug( "Actual: " + str( current ) )
2885 getResults = main.FALSE
2886 else:
2887 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002888 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002889 " has repeat elements in" +
2890 " set " + onosSetName + ":\n" +
2891 str( getResponses[ i ] ) )
2892 getResults = main.FALSE
2893 elif getResponses[ i ] == main.ERROR:
2894 getResults = main.FALSE
2895 sizeResponses = []
2896 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002897 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002898 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002899 name="setTestSize-" + str( i ),
2900 args=[ onosSetName ] )
2901 threads.append( t )
2902 t.start()
2903 for t in threads:
2904 t.join()
2905 sizeResponses.append( t.result )
2906 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002907 for i in range( len( main.activeNodes ) ):
2908 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002909 if size != sizeResponses[ i ]:
2910 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07002911 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002912 " expected a size of " + str( size ) +
2913 " for set " + onosSetName +
2914 " but got " + str( sizeResponses[ i ] ) )
2915 removeResults = removeResults and getResults and sizeResults
2916 utilities.assert_equals( expect=main.TRUE,
2917 actual=removeResults,
2918 onpass="Set remove correct",
2919 onfail="Set remove was incorrect" )
2920
2921 main.step( "Distributed Set removeAll()" )
2922 onosSet.difference_update( addAllValue.split() )
2923 removeAllResponses = []
2924 threads = []
2925 try:
Jon Halla440e872016-03-31 15:15:50 -07002926 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002927 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07002928 name="setTestRemoveAll-" + str( i ),
2929 args=[ onosSetName, addAllValue ] )
2930 threads.append( t )
2931 t.start()
2932 for t in threads:
2933 t.join()
2934 removeAllResponses.append( t.result )
2935 except Exception, e:
2936 main.log.exception(e)
2937
2938 # main.TRUE = successfully changed the set
2939 # main.FALSE = action resulted in no change in set
2940 # main.ERROR - Some error in executing the function
2941 removeAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002942 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07002943 if removeAllResponses[ i ] == main.TRUE:
2944 # All is well
2945 pass
2946 elif removeAllResponses[ i ] == main.FALSE:
2947 # not in set, probably fine
2948 pass
2949 elif removeAllResponses[ i ] == main.ERROR:
2950 # Error in execution
2951 removeAllResults = main.FALSE
2952 else:
2953 # unexpected result
2954 removeAllResults = main.FALSE
2955 if removeAllResults != main.TRUE:
2956 main.log.error( "Error executing set removeAll" )
2957
2958 # Check if set is still correct
2959 size = len( onosSet )
2960 getResponses = []
2961 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002962 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002963 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07002964 name="setTestGet-" + str( i ),
2965 args=[ onosSetName ] )
2966 threads.append( t )
2967 t.start()
2968 for t in threads:
2969 t.join()
2970 getResponses.append( t.result )
2971 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07002972 for i in range( len( main.activeNodes ) ):
2973 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07002974 if isinstance( getResponses[ i ], list):
2975 current = set( getResponses[ i ] )
2976 if len( current ) == len( getResponses[ i ] ):
2977 # no repeats
2978 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07002979 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002980 " has incorrect view" +
2981 " of set " + onosSetName + ":\n" +
2982 str( getResponses[ i ] ) )
2983 main.log.debug( "Expected: " + str( onosSet ) )
2984 main.log.debug( "Actual: " + str( current ) )
2985 getResults = main.FALSE
2986 else:
2987 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07002988 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07002989 " has repeat elements in" +
2990 " set " + onosSetName + ":\n" +
2991 str( getResponses[ i ] ) )
2992 getResults = main.FALSE
2993 elif getResponses[ i ] == main.ERROR:
2994 getResults = main.FALSE
2995 sizeResponses = []
2996 threads = []
Jon Halla440e872016-03-31 15:15:50 -07002997 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07002998 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07002999 name="setTestSize-" + str( i ),
3000 args=[ onosSetName ] )
3001 threads.append( t )
3002 t.start()
3003 for t in threads:
3004 t.join()
3005 sizeResponses.append( t.result )
3006 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003007 for i in range( len( main.activeNodes ) ):
3008 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003009 if size != sizeResponses[ i ]:
3010 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003011 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003012 " expected a size of " + str( size ) +
3013 " for set " + onosSetName +
3014 " but got " + str( sizeResponses[ i ] ) )
3015 removeAllResults = removeAllResults and getResults and sizeResults
3016 utilities.assert_equals( expect=main.TRUE,
3017 actual=removeAllResults,
3018 onpass="Set removeAll correct",
3019 onfail="Set removeAll was incorrect" )
3020
3021 main.step( "Distributed Set addAll()" )
3022 onosSet.update( addAllValue.split() )
3023 addResponses = []
3024 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003025 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003026 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003027 name="setTestAddAll-" + str( i ),
3028 args=[ onosSetName, addAllValue ] )
3029 threads.append( t )
3030 t.start()
3031 for t in threads:
3032 t.join()
3033 addResponses.append( t.result )
3034
3035 # main.TRUE = successfully changed the set
3036 # main.FALSE = action resulted in no change in set
3037 # main.ERROR - Some error in executing the function
3038 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003039 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07003040 if addResponses[ i ] == main.TRUE:
3041 # All is well
3042 pass
3043 elif addResponses[ i ] == main.FALSE:
3044 # Already in set, probably fine
3045 pass
3046 elif addResponses[ i ] == main.ERROR:
3047 # Error in execution
3048 addAllResults = main.FALSE
3049 else:
3050 # unexpected result
3051 addAllResults = main.FALSE
3052 if addAllResults != main.TRUE:
3053 main.log.error( "Error executing set addAll" )
3054
3055 # Check if set is still correct
3056 size = len( onosSet )
3057 getResponses = []
3058 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003059 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003060 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003061 name="setTestGet-" + str( i ),
3062 args=[ onosSetName ] )
3063 threads.append( t )
3064 t.start()
3065 for t in threads:
3066 t.join()
3067 getResponses.append( t.result )
3068 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003069 for i in range( len( main.activeNodes ) ):
3070 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003071 if isinstance( getResponses[ i ], list):
3072 current = set( getResponses[ i ] )
3073 if len( current ) == len( getResponses[ i ] ):
3074 # no repeats
3075 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07003076 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003077 " has incorrect view" +
3078 " of set " + onosSetName + ":\n" +
3079 str( getResponses[ i ] ) )
3080 main.log.debug( "Expected: " + str( onosSet ) )
3081 main.log.debug( "Actual: " + str( current ) )
3082 getResults = main.FALSE
3083 else:
3084 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07003085 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003086 " has repeat elements in" +
3087 " set " + onosSetName + ":\n" +
3088 str( getResponses[ i ] ) )
3089 getResults = main.FALSE
3090 elif getResponses[ i ] == main.ERROR:
3091 getResults = main.FALSE
3092 sizeResponses = []
3093 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003094 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003095 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003096 name="setTestSize-" + str( i ),
3097 args=[ onosSetName ] )
3098 threads.append( t )
3099 t.start()
3100 for t in threads:
3101 t.join()
3102 sizeResponses.append( t.result )
3103 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003104 for i in range( len( main.activeNodes ) ):
3105 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003106 if size != sizeResponses[ i ]:
3107 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003108 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003109 " expected a size of " + str( size ) +
3110 " for set " + onosSetName +
3111 " but got " + str( sizeResponses[ i ] ) )
3112 addAllResults = addAllResults and getResults and sizeResults
3113 utilities.assert_equals( expect=main.TRUE,
3114 actual=addAllResults,
3115 onpass="Set addAll correct",
3116 onfail="Set addAll was incorrect" )
3117
3118 main.step( "Distributed Set clear()" )
3119 onosSet.clear()
3120 clearResponses = []
3121 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003122 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003123 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003124 name="setTestClear-" + str( i ),
3125 args=[ onosSetName, " "], # Values doesn't matter
3126 kwargs={ "clear": True } )
3127 threads.append( t )
3128 t.start()
3129 for t in threads:
3130 t.join()
3131 clearResponses.append( t.result )
3132
3133 # main.TRUE = successfully changed the set
3134 # main.FALSE = action resulted in no change in set
3135 # main.ERROR - Some error in executing the function
3136 clearResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003137 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07003138 if clearResponses[ i ] == main.TRUE:
3139 # All is well
3140 pass
3141 elif clearResponses[ i ] == main.FALSE:
3142 # Nothing set, probably fine
3143 pass
3144 elif clearResponses[ i ] == main.ERROR:
3145 # Error in execution
3146 clearResults = main.FALSE
3147 else:
3148 # unexpected result
3149 clearResults = main.FALSE
3150 if clearResults != main.TRUE:
3151 main.log.error( "Error executing set clear" )
3152
3153 # Check if set is still correct
3154 size = len( onosSet )
3155 getResponses = []
3156 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003157 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003158 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003159 name="setTestGet-" + str( i ),
3160 args=[ onosSetName ] )
3161 threads.append( t )
3162 t.start()
3163 for t in threads:
3164 t.join()
3165 getResponses.append( t.result )
3166 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003167 for i in range( len( main.activeNodes ) ):
3168 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003169 if isinstance( getResponses[ i ], list):
3170 current = set( getResponses[ i ] )
3171 if len( current ) == len( getResponses[ i ] ):
3172 # no repeats
3173 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07003174 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003175 " has incorrect view" +
3176 " of set " + onosSetName + ":\n" +
3177 str( getResponses[ i ] ) )
3178 main.log.debug( "Expected: " + str( onosSet ) )
3179 main.log.debug( "Actual: " + str( current ) )
3180 getResults = main.FALSE
3181 else:
3182 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07003183 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003184 " has repeat elements in" +
3185 " set " + onosSetName + ":\n" +
3186 str( getResponses[ i ] ) )
3187 getResults = main.FALSE
3188 elif getResponses[ i ] == main.ERROR:
3189 getResults = main.FALSE
3190 sizeResponses = []
3191 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003192 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003193 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003194 name="setTestSize-" + str( i ),
3195 args=[ onosSetName ] )
3196 threads.append( t )
3197 t.start()
3198 for t in threads:
3199 t.join()
3200 sizeResponses.append( t.result )
3201 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003202 for i in range( len( main.activeNodes ) ):
3203 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003204 if size != sizeResponses[ i ]:
3205 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003206 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003207 " expected a size of " + str( size ) +
3208 " for set " + onosSetName +
3209 " but got " + str( sizeResponses[ i ] ) )
3210 clearResults = clearResults and getResults and sizeResults
3211 utilities.assert_equals( expect=main.TRUE,
3212 actual=clearResults,
3213 onpass="Set clear correct",
3214 onfail="Set clear was incorrect" )
3215
3216 main.step( "Distributed Set addAll()" )
3217 onosSet.update( addAllValue.split() )
3218 addResponses = []
3219 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003220 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003221 t = main.Thread( target=main.CLIs[i].setTestAdd,
Jon Hall85794ff2015-07-08 14:12:30 -07003222 name="setTestAddAll-" + str( i ),
3223 args=[ onosSetName, addAllValue ] )
3224 threads.append( t )
3225 t.start()
3226 for t in threads:
3227 t.join()
3228 addResponses.append( t.result )
3229
3230 # main.TRUE = successfully changed the set
3231 # main.FALSE = action resulted in no change in set
3232 # main.ERROR - Some error in executing the function
3233 addAllResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003234 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07003235 if addResponses[ i ] == main.TRUE:
3236 # All is well
3237 pass
3238 elif addResponses[ i ] == main.FALSE:
3239 # Already in set, probably fine
3240 pass
3241 elif addResponses[ i ] == main.ERROR:
3242 # Error in execution
3243 addAllResults = main.FALSE
3244 else:
3245 # unexpected result
3246 addAllResults = main.FALSE
3247 if addAllResults != main.TRUE:
3248 main.log.error( "Error executing set addAll" )
3249
3250 # Check if set is still correct
3251 size = len( onosSet )
3252 getResponses = []
3253 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003254 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003255 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003256 name="setTestGet-" + str( i ),
3257 args=[ onosSetName ] )
3258 threads.append( t )
3259 t.start()
3260 for t in threads:
3261 t.join()
3262 getResponses.append( t.result )
3263 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003264 for i in range( len( main.activeNodes ) ):
3265 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003266 if isinstance( getResponses[ i ], list):
3267 current = set( getResponses[ i ] )
3268 if len( current ) == len( getResponses[ i ] ):
3269 # no repeats
3270 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07003271 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003272 " has incorrect view" +
3273 " of set " + onosSetName + ":\n" +
3274 str( getResponses[ i ] ) )
3275 main.log.debug( "Expected: " + str( onosSet ) )
3276 main.log.debug( "Actual: " + str( current ) )
3277 getResults = main.FALSE
3278 else:
3279 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07003280 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003281 " has repeat elements in" +
3282 " set " + onosSetName + ":\n" +
3283 str( getResponses[ i ] ) )
3284 getResults = main.FALSE
3285 elif getResponses[ i ] == main.ERROR:
3286 getResults = main.FALSE
3287 sizeResponses = []
3288 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003289 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003290 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003291 name="setTestSize-" + str( i ),
3292 args=[ onosSetName ] )
3293 threads.append( t )
3294 t.start()
3295 for t in threads:
3296 t.join()
3297 sizeResponses.append( t.result )
3298 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003299 for i in range( len( main.activeNodes ) ):
3300 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003301 if size != sizeResponses[ i ]:
3302 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003303 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003304 " expected a size of " + str( size ) +
3305 " for set " + onosSetName +
3306 " but got " + str( sizeResponses[ i ] ) )
3307 addAllResults = addAllResults and getResults and sizeResults
3308 utilities.assert_equals( expect=main.TRUE,
3309 actual=addAllResults,
3310 onpass="Set addAll correct",
3311 onfail="Set addAll was incorrect" )
3312
3313 main.step( "Distributed Set retain()" )
3314 onosSet.intersection_update( retainValue.split() )
3315 retainResponses = []
3316 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003317 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003318 t = main.Thread( target=main.CLIs[i].setTestRemove,
Jon Hall85794ff2015-07-08 14:12:30 -07003319 name="setTestRetain-" + str( i ),
3320 args=[ onosSetName, retainValue ],
3321 kwargs={ "retain": True } )
3322 threads.append( t )
3323 t.start()
3324 for t in threads:
3325 t.join()
3326 retainResponses.append( t.result )
3327
3328 # main.TRUE = successfully changed the set
3329 # main.FALSE = action resulted in no change in set
3330 # main.ERROR - Some error in executing the function
3331 retainResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003332 for i in range( len( main.activeNodes ) ):
Jon Hall85794ff2015-07-08 14:12:30 -07003333 if retainResponses[ i ] == main.TRUE:
3334 # All is well
3335 pass
3336 elif retainResponses[ i ] == main.FALSE:
3337 # Already in set, probably fine
3338 pass
3339 elif retainResponses[ i ] == main.ERROR:
3340 # Error in execution
3341 retainResults = main.FALSE
3342 else:
3343 # unexpected result
3344 retainResults = main.FALSE
3345 if retainResults != main.TRUE:
3346 main.log.error( "Error executing set retain" )
3347
3348 # Check if set is still correct
3349 size = len( onosSet )
3350 getResponses = []
3351 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003352 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003353 t = main.Thread( target=main.CLIs[i].setTestGet,
Jon Hall85794ff2015-07-08 14:12:30 -07003354 name="setTestGet-" + str( i ),
3355 args=[ onosSetName ] )
3356 threads.append( t )
3357 t.start()
3358 for t in threads:
3359 t.join()
3360 getResponses.append( t.result )
3361 getResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003362 for i in range( len( main.activeNodes ) ):
3363 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003364 if isinstance( getResponses[ i ], list):
3365 current = set( getResponses[ i ] )
3366 if len( current ) == len( getResponses[ i ] ):
3367 # no repeats
3368 if onosSet != current:
Jon Halla440e872016-03-31 15:15:50 -07003369 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003370 " has incorrect view" +
3371 " of set " + onosSetName + ":\n" +
3372 str( getResponses[ i ] ) )
3373 main.log.debug( "Expected: " + str( onosSet ) )
3374 main.log.debug( "Actual: " + str( current ) )
3375 getResults = main.FALSE
3376 else:
3377 # error, set is not a set
Jon Halla440e872016-03-31 15:15:50 -07003378 main.log.error( "ONOS" + node +
Jon Hall85794ff2015-07-08 14:12:30 -07003379 " has repeat elements in" +
3380 " set " + onosSetName + ":\n" +
3381 str( getResponses[ i ] ) )
3382 getResults = main.FALSE
3383 elif getResponses[ i ] == main.ERROR:
3384 getResults = main.FALSE
3385 sizeResponses = []
3386 threads = []
Jon Halla440e872016-03-31 15:15:50 -07003387 for i in main.activeNodes:
Jon Halle1a3b752015-07-22 13:02:46 -07003388 t = main.Thread( target=main.CLIs[i].setTestSize,
Jon Hall85794ff2015-07-08 14:12:30 -07003389 name="setTestSize-" + str( i ),
3390 args=[ onosSetName ] )
3391 threads.append( t )
3392 t.start()
3393 for t in threads:
3394 t.join()
3395 sizeResponses.append( t.result )
3396 sizeResults = main.TRUE
Jon Halla440e872016-03-31 15:15:50 -07003397 for i in range( len( main.activeNodes ) ):
3398 node = str( main.activeNodes[i] + 1 )
Jon Hall85794ff2015-07-08 14:12:30 -07003399 if size != sizeResponses[ i ]:
3400 sizeResults = main.FALSE
Jon Halla440e872016-03-31 15:15:50 -07003401 main.log.error( "ONOS" + node + " expected a size of " +
Jon Hall85794ff2015-07-08 14:12:30 -07003402 str( size ) + " for set " + onosSetName +
3403 " but got " + str( sizeResponses[ i ] ) )
3404 retainResults = retainResults and getResults and sizeResults
3405 utilities.assert_equals( expect=main.TRUE,
3406 actual=retainResults,
3407 onpass="Set retain correct",
3408 onfail="Set retain was incorrect" )
3409
Jon Hall2a5002c2015-08-21 16:49:11 -07003410 # Transactional maps
3411 main.step( "Partitioned Transactional maps put" )
3412 tMapValue = "Testing"
3413 numKeys = 100
3414 putResult = True
Jon Halla440e872016-03-31 15:15:50 -07003415 node = main.activeNodes[0]
3416 putResponses = main.CLIs[node].transactionalMapPut( numKeys, tMapValue )
3417 if putResponses and len( putResponses ) == 100:
Jon Hall2a5002c2015-08-21 16:49:11 -07003418 for i in putResponses:
3419 if putResponses[ i ][ 'value' ] != tMapValue:
3420 putResult = False
3421 else:
3422 putResult = False
3423 if not putResult:
3424 main.log.debug( "Put response values: " + str( putResponses ) )
3425 utilities.assert_equals( expect=True,
3426 actual=putResult,
3427 onpass="Partitioned Transactional Map put successful",
3428 onfail="Partitioned Transactional Map put values are incorrect" )
3429
3430 main.step( "Partitioned Transactional maps get" )
Jon Hall9bfadd22016-05-11 14:48:07 -07003431 # FIXME: is this sleep needed?
3432 time.sleep( 5 )
3433
Jon Hall2a5002c2015-08-21 16:49:11 -07003434 getCheck = True
3435 for n in range( 1, numKeys + 1 ):
3436 getResponses = []
3437 threads = []
3438 valueCheck = True
Jon Halla440e872016-03-31 15:15:50 -07003439 for i in main.activeNodes:
Jon Hall2a5002c2015-08-21 16:49:11 -07003440 t = main.Thread( target=main.CLIs[i].transactionalMapGet,
3441 name="TMap-get-" + str( i ),
Jon Halla440e872016-03-31 15:15:50 -07003442 args=[ "Key" + str( n ) ] )
Jon Hall2a5002c2015-08-21 16:49:11 -07003443 threads.append( t )
3444 t.start()
3445 for t in threads:
3446 t.join()
3447 getResponses.append( t.result )
3448 for node in getResponses:
3449 if node != tMapValue:
3450 valueCheck = False
3451 if not valueCheck:
3452 main.log.warn( "Values for key 'Key" + str( n ) + "' do not match:" )
3453 main.log.warn( getResponses )
3454 getCheck = getCheck and valueCheck
3455 utilities.assert_equals( expect=True,
3456 actual=getCheck,
3457 onpass="Partitioned Transactional Map get values were correct",
3458 onfail="Partitioned Transactional Map values incorrect" )