Package TestON :: Package tests :: Package HAsanity :: Module HAsanity
[hide private]
[frames] | no frames]

Source Code for Module TestON.tests.HAsanity.HAsanity

   1  """ 
   2  Description: This test is to determine if the HA test setup is 
   3      working correctly. There are no failures so this test should 
   4      have a 100% pass rate 
   5   
   6  List of test cases: 
   7  CASE1: Compile ONOS and push it to the test machines 
   8  CASE2: Assign devices to controllers 
   9  CASE21: Assign mastership to controllers 
  10  CASE3: Assign intents 
  11  CASE4: Ping across added host intents 
  12  CASE5: Reading state of ONOS 
  13  CASE6: The Failure case. Since this is the Sanity test, we do nothing. 
  14  CASE7: Check state after control plane failure 
  15  CASE8: Compare topo 
  16  CASE9: Link s3-s28 down 
  17  CASE10: Link s3-s28 up 
  18  CASE11: Switch down 
  19  CASE12: Switch up 
  20  CASE13: Clean up 
  21  CASE14: start election app on all onos nodes 
  22  CASE15: Check that Leadership Election is still functional 
  23  CASE16: Install Distributed Primitives app 
  24  CASE17: Check for basic functionality with distributed primitives 
  25  """ 
  26   
  27   
28 -class HAsanity:
29
30 - def __init__( self ):
31 self.default = ''
32
33 - def CASE1( self, main ):
34 """ 35 CASE1 is to compile ONOS and push it to the test machines 36 37 Startup sequence: 38 cell <name> 39 onos-verify-cell 40 NOTE: temporary - onos-remove-raft-logs 41 onos-uninstall 42 start mininet 43 git pull 44 mvn clean install 45 onos-package 46 onos-install -f 47 onos-wait-for-start 48 start cli sessions 49 start tcpdump 50 """ 51 import imp 52 main.log.info( "ONOS HA Sanity test - initialization" ) 53 main.case( "Setting up test environment" ) 54 main.caseExplanation = "Setup the test environment including " +\ 55 "installing ONOS, starting Mininet and ONOS" +\ 56 "cli sessions." 57 # TODO: save all the timers and output them for plotting 58 59 # load some variables from the params file 60 PULLCODE = False 61 if main.params[ 'Git' ] == 'True': 62 PULLCODE = True 63 gitBranch = main.params[ 'branch' ] 64 cellName = main.params[ 'ENV' ][ 'cellName' ] 65 66 main.numCtrls = int( main.params[ 'num_controllers' ] ) 67 if main.ONOSbench.maxNodes: 68 if main.ONOSbench.maxNodes < main.numCtrls: 69 main.numCtrls = int( main.ONOSbench.maxNodes ) 70 # TODO: refactor how to get onos port, maybe put into component tag? 71 # set global variables 72 global ONOS1Port 73 global ONOS2Port 74 global ONOS3Port 75 global ONOS4Port 76 global ONOS5Port 77 global ONOS6Port 78 global ONOS7Port 79 80 # FIXME: just get controller port from params? 81 # TODO: do we really need all these? 82 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ] 83 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ] 84 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ] 85 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ] 86 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ] 87 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ] 88 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ] 89 90 try: 91 fileName = "Counters" 92 path = main.params[ 'imports' ][ 'path' ] 93 main.Counters = imp.load_source( fileName, 94 path + fileName + ".py" ) 95 except Exception as e: 96 main.log.exception( e ) 97 main.cleanup() 98 main.exit() 99 100 main.CLIs = [] 101 main.nodes = [] 102 ipList = [] 103 for i in range( 1, main.numCtrls + 1 ): 104 try: 105 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) ) 106 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) ) 107 ipList.append( main.nodes[ -1 ].ip_address ) 108 except AttributeError: 109 break 110 111 main.step( "Create cell file" ) 112 cellAppString = main.params[ 'ENV' ][ 'appString' ] 113 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName, 114 main.Mininet1.ip_address, 115 cellAppString, ipList ) 116 main.step( "Applying cell variable to environment" ) 117 cellResult = main.ONOSbench.setCell( cellName ) 118 verifyResult = main.ONOSbench.verifyCell() 119 120 # FIXME:this is short term fix 121 main.log.info( "Removing raft logs" ) 122 main.ONOSbench.onosRemoveRaftLogs() 123 124 main.log.info( "Uninstalling ONOS" ) 125 for node in main.nodes: 126 main.ONOSbench.onosUninstall( node.ip_address ) 127 128 # Make sure ONOS is DEAD 129 main.log.info( "Killing any ONOS processes" ) 130 killResults = main.TRUE 131 for node in main.nodes: 132 killed = main.ONOSbench.onosKill( node.ip_address ) 133 killResults = killResults and killed 134 135 cleanInstallResult = main.TRUE 136 gitPullResult = main.TRUE 137 138 main.step( "Starting Mininet" ) 139 # scp topo file to mininet 140 # TODO: move to params? 141 topoName = "obelisk.py" 142 filePath = main.ONOSbench.home + "/tools/test/topos/" 143 main.ONOSbench.copyMininetFile( topoName, filePath, 144 main.Mininet1.user_name, 145 main.Mininet1.ip_address ) 146 mnResult = main.Mininet1.startNet( ) 147 utilities.assert_equals( expect=main.TRUE, actual=mnResult, 148 onpass="Mininet Started", 149 onfail="Error starting Mininet" ) 150 151 main.step( "Git checkout and pull " + gitBranch ) 152 if PULLCODE: 153 main.ONOSbench.gitCheckout( gitBranch ) 154 gitPullResult = main.ONOSbench.gitPull() 155 # values of 1 or 3 are good 156 utilities.assert_lesser( expect=0, actual=gitPullResult, 157 onpass="Git pull successful", 158 onfail="Git pull failed" ) 159 main.ONOSbench.getVersion( report=True ) 160 161 main.step( "Using mvn clean install" ) 162 cleanInstallResult = main.TRUE 163 if PULLCODE and gitPullResult == main.TRUE: 164 cleanInstallResult = main.ONOSbench.cleanInstall() 165 else: 166 main.log.warn( "Did not pull new code so skipping mvn " + 167 "clean install" ) 168 utilities.assert_equals( expect=main.TRUE, 169 actual=cleanInstallResult, 170 onpass="MCI successful", 171 onfail="MCI failed" ) 172 # GRAPHS 173 # NOTE: important params here: 174 # job = name of Jenkins job 175 # Plot Name = Plot-HA, only can be used if multiple plots 176 # index = The number of the graph under plot name 177 job = "HAsanity" 178 plotName = "Plot-HA" 179 graphs = '<ac:structured-macro ac:name="html">\n' 180 graphs += '<ac:plain-text-body><![CDATA[\n' 181 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\ 182 '/plot/' + plotName + '/getPlot?index=0' +\ 183 '&width=500&height=300"' +\ 184 'noborder="0" width="500" height="300" scrolling="yes" ' +\ 185 'seamless="seamless"></iframe>\n' 186 graphs += ']]></ac:plain-text-body>\n' 187 graphs += '</ac:structured-macro>\n' 188 main.log.wiki(graphs) 189 190 main.step( "Creating ONOS package" ) 191 packageResult = main.ONOSbench.onosPackage() 192 utilities.assert_equals( expect=main.TRUE, actual=packageResult, 193 onpass="ONOS package successful", 194 onfail="ONOS package failed" ) 195 196 main.step( "Installing ONOS package" ) 197 onosInstallResult = main.TRUE 198 for node in main.nodes: 199 tmpResult = main.ONOSbench.onosInstall( options="-f", 200 node=node.ip_address ) 201 onosInstallResult = onosInstallResult and tmpResult 202 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult, 203 onpass="ONOS install successful", 204 onfail="ONOS install failed" ) 205 206 main.step( "Checking if ONOS is up yet" ) 207 for i in range( 2 ): 208 onosIsupResult = main.TRUE 209 for node in main.nodes: 210 started = main.ONOSbench.isup( node.ip_address ) 211 if not started: 212 main.log.error( node.name + " didn't start!" ) 213 main.ONOSbench.onosStop( node.ip_address ) 214 main.ONOSbench.onosStart( node.ip_address ) 215 onosIsupResult = onosIsupResult and started 216 if onosIsupResult == main.TRUE: 217 break 218 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult, 219 onpass="ONOS startup successful", 220 onfail="ONOS startup failed" ) 221 222 main.log.step( "Starting ONOS CLI sessions" ) 223 cliResults = main.TRUE 224 threads = [] 225 for i in range( main.numCtrls ): 226 t = main.Thread( target=main.CLIs[i].startOnosCli, 227 name="startOnosCli-" + str( i ), 228 args=[main.nodes[i].ip_address] ) 229 threads.append( t ) 230 t.start() 231 232 for t in threads: 233 t.join() 234 cliResults = cliResults and t.result 235 utilities.assert_equals( expect=main.TRUE, actual=cliResults, 236 onpass="ONOS cli startup successful", 237 onfail="ONOS cli startup failed" ) 238 239 if main.params[ 'tcpdump' ].lower() == "true": 240 main.step( "Start Packet Capture MN" ) 241 main.Mininet2.startTcpdump( 242 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST ) 243 + "-MN.pcap", 244 intf=main.params[ 'MNtcpdump' ][ 'intf' ], 245 port=main.params[ 'MNtcpdump' ][ 'port' ] ) 246 247 main.step( "App Ids check" ) 248 appCheck = main.TRUE 249 threads = [] 250 for i in range( main.numCtrls ): 251 t = main.Thread( target=main.CLIs[i].appToIDCheck, 252 name="appToIDCheck-" + str( i ), 253 args=[] ) 254 threads.append( t ) 255 t.start() 256 257 for t in threads: 258 t.join() 259 appCheck = appCheck and t.result 260 if appCheck != main.TRUE: 261 main.log.warn( main.CLIs[0].apps() ) 262 main.log.warn( main.CLIs[0].appIDs() ) 263 utilities.assert_equals( expect=main.TRUE, actual=appCheck, 264 onpass="App Ids seem to be correct", 265 onfail="Something is wrong with app Ids" ) 266 267 if cliResults == main.FALSE: 268 main.log.error( "Failed to start ONOS, stopping test" ) 269 main.cleanup() 270 main.exit()
271
272 - def CASE2( self, main ):
273 """ 274 Assign devices to controllers 275 """ 276 import re 277 assert main.numCtrls, "main.numCtrls not defined" 278 assert main, "main not defined" 279 assert utilities.assert_equals, "utilities.assert_equals not defined" 280 assert main.CLIs, "main.CLIs not defined" 281 assert main.nodes, "main.nodes not defined" 282 assert ONOS1Port, "ONOS1Port not defined" 283 assert ONOS2Port, "ONOS2Port not defined" 284 assert ONOS3Port, "ONOS3Port not defined" 285 assert ONOS4Port, "ONOS4Port not defined" 286 assert ONOS5Port, "ONOS5Port not defined" 287 assert ONOS6Port, "ONOS6Port not defined" 288 assert ONOS7Port, "ONOS7Port not defined" 289 290 main.case( "Assigning devices to controllers" ) 291 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\ 292 "and check that an ONOS node becomes the " +\ 293 "master of the device." 294 main.step( "Assign switches to controllers" ) 295 296 ipList = [] 297 for i in range( main.numCtrls ): 298 ipList.append( main.nodes[ i ].ip_address ) 299 swList = [] 300 for i in range( 1, 29 ): 301 swList.append( "s" + str( i ) ) 302 main.Mininet1.assignSwController( sw=swList, ip=ipList ) 303 304 mastershipCheck = main.TRUE 305 for i in range( 1, 29 ): 306 response = main.Mininet1.getSwController( "s" + str( i ) ) 307 try: 308 main.log.info( str( response ) ) 309 except Exception: 310 main.log.info( repr( response ) ) 311 for node in main.nodes: 312 if re.search( "tcp:" + node.ip_address, response ): 313 mastershipCheck = mastershipCheck and main.TRUE 314 else: 315 main.log.error( "Error, node " + node.ip_address + " is " + 316 "not in the list of controllers s" + 317 str( i ) + " is connecting to." ) 318 mastershipCheck = main.FALSE 319 utilities.assert_equals( 320 expect=main.TRUE, 321 actual=mastershipCheck, 322 onpass="Switch mastership assigned correctly", 323 onfail="Switches not assigned correctly to controllers" )
324
325 - def CASE21( self, main ):
326 """ 327 Assign mastership to controllers 328 """ 329 import time 330 assert main.numCtrls, "main.numCtrls not defined" 331 assert main, "main not defined" 332 assert utilities.assert_equals, "utilities.assert_equals not defined" 333 assert main.CLIs, "main.CLIs not defined" 334 assert main.nodes, "main.nodes not defined" 335 assert ONOS1Port, "ONOS1Port not defined" 336 assert ONOS2Port, "ONOS2Port not defined" 337 assert ONOS3Port, "ONOS3Port not defined" 338 assert ONOS4Port, "ONOS4Port not defined" 339 assert ONOS5Port, "ONOS5Port not defined" 340 assert ONOS6Port, "ONOS6Port not defined" 341 assert ONOS7Port, "ONOS7Port not defined" 342 343 main.case( "Assigning Controller roles for switches" ) 344 main.caseExplanation = "Check that ONOS is connected to each " +\ 345 "device. Then manually assign" +\ 346 " mastership to specific ONOS nodes using" +\ 347 " 'device-role'" 348 main.step( "Assign mastership of switches to specific controllers" ) 349 # Manually assign mastership to the controller we want 350 roleCall = main.TRUE 351 352 ipList = [ ] 353 deviceList = [] 354 try: 355 # Assign mastership to specific controllers. This assignment was 356 # determined for a 7 node cluser, but will work with any sized 357 # cluster 358 for i in range( 1, 29 ): # switches 1 through 28 359 # set up correct variables: 360 if i == 1: 361 c = 0 362 ip = main.nodes[ c ].ip_address # ONOS1 363 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' ) 364 elif i == 2: 365 c = 1 % main.numCtrls 366 ip = main.nodes[ c ].ip_address # ONOS2 367 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' ) 368 elif i == 3: 369 c = 1 % main.numCtrls 370 ip = main.nodes[ c ].ip_address # ONOS2 371 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' ) 372 elif i == 4: 373 c = 3 % main.numCtrls 374 ip = main.nodes[ c ].ip_address # ONOS4 375 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' ) 376 elif i == 5: 377 c = 2 % main.numCtrls 378 ip = main.nodes[ c ].ip_address # ONOS3 379 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' ) 380 elif i == 6: 381 c = 2 % main.numCtrls 382 ip = main.nodes[ c ].ip_address # ONOS3 383 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' ) 384 elif i == 7: 385 c = 5 % main.numCtrls 386 ip = main.nodes[ c ].ip_address # ONOS6 387 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' ) 388 elif i >= 8 and i <= 17: 389 c = 4 % main.numCtrls 390 ip = main.nodes[ c ].ip_address # ONOS5 391 dpid = '3' + str( i ).zfill( 3 ) 392 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' ) 393 elif i >= 18 and i <= 27: 394 c = 6 % main.numCtrls 395 ip = main.nodes[ c ].ip_address # ONOS7 396 dpid = '6' + str( i ).zfill( 3 ) 397 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' ) 398 elif i == 28: 399 c = 0 400 ip = main.nodes[ c ].ip_address # ONOS1 401 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' ) 402 else: 403 main.log.error( "You didn't write an else statement for " + 404 "switch s" + str( i ) ) 405 roleCall = main.FALSE 406 # Assign switch 407 assert deviceId, "No device id for s" + str( i ) + " in ONOS" 408 # TODO: make this controller dynamic 409 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId, 410 ip ) 411 ipList.append( ip ) 412 deviceList.append( deviceId ) 413 except ( AttributeError, AssertionError ): 414 main.log.exception( "Something is wrong with ONOS device view" ) 415 main.log.info( main.ONOScli1.devices() ) 416 utilities.assert_equals( 417 expect=main.TRUE, 418 actual=roleCall, 419 onpass="Re-assigned switch mastership to designated controller", 420 onfail="Something wrong with deviceRole calls" ) 421 422 main.step( "Check mastership was correctly assigned" ) 423 roleCheck = main.TRUE 424 # NOTE: This is due to the fact that device mastership change is not 425 # atomic and is actually a multi step process 426 time.sleep( 5 ) 427 for i in range( len( ipList ) ): 428 ip = ipList[i] 429 deviceId = deviceList[i] 430 # Check assignment 431 master = main.ONOScli1.getRole( deviceId ).get( 'master' ) 432 if ip in master: 433 roleCheck = roleCheck and main.TRUE 434 else: 435 roleCheck = roleCheck and main.FALSE 436 main.log.error( "Error, controller " + ip + " is not" + 437 " master " + "of device " + 438 str( deviceId ) + ". Master is " + 439 repr( master ) + "." ) 440 utilities.assert_equals( 441 expect=main.TRUE, 442 actual=roleCheck, 443 onpass="Switches were successfully reassigned to designated " + 444 "controller", 445 onfail="Switches were not successfully reassigned" )
446
447 - def CASE3( self, main ):
448 """ 449 Assign intents 450 """ 451 import time 452 import json 453 assert main.numCtrls, "main.numCtrls not defined" 454 assert main, "main not defined" 455 assert utilities.assert_equals, "utilities.assert_equals not defined" 456 assert main.CLIs, "main.CLIs not defined" 457 assert main.nodes, "main.nodes not defined" 458 main.case( "Adding host Intents" ) 459 main.caseExplanation = "Discover hosts by using pingall then " +\ 460 "assign predetermined host-to-host intents." +\ 461 " After installation, check that the intent" +\ 462 " is distributed to all nodes and the state" +\ 463 " is INSTALLED" 464 465 # install onos-app-fwd 466 main.step( "Install reactive forwarding app" ) 467 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" ) 468 utilities.assert_equals( expect=main.TRUE, actual=installResults, 469 onpass="Install fwd successful", 470 onfail="Install fwd failed" ) 471 472 main.step( "Check app ids" ) 473 appCheck = main.TRUE 474 threads = [] 475 for i in range( main.numCtrls ): 476 t = main.Thread( target=main.CLIs[i].appToIDCheck, 477 name="appToIDCheck-" + str( i ), 478 args=[] ) 479 threads.append( t ) 480 t.start() 481 482 for t in threads: 483 t.join() 484 appCheck = appCheck and t.result 485 if appCheck != main.TRUE: 486 main.log.warn( main.CLIs[0].apps() ) 487 main.log.warn( main.CLIs[0].appIDs() ) 488 utilities.assert_equals( expect=main.TRUE, actual=appCheck, 489 onpass="App Ids seem to be correct", 490 onfail="Something is wrong with app Ids" ) 491 492 main.step( "Discovering Hosts( Via pingall for now )" ) 493 # FIXME: Once we have a host discovery mechanism, use that instead 494 # REACTIVE FWD test 495 pingResult = main.FALSE 496 for i in range(2): # Retry if pingall fails first time 497 time1 = time.time() 498 pingResult = main.Mininet1.pingall() 499 if i == 0: 500 utilities.assert_equals( 501 expect=main.TRUE, 502 actual=pingResult, 503 onpass="Reactive Pingall test passed", 504 onfail="Reactive Pingall failed, " + 505 "one or more ping pairs failed" ) 506 time2 = time.time() 507 main.log.info( "Time for pingall: %2f seconds" % 508 ( time2 - time1 ) ) 509 # timeout for fwd flows 510 time.sleep( 11 ) 511 # uninstall onos-app-fwd 512 main.step( "Uninstall reactive forwarding app" ) 513 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" ) 514 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult, 515 onpass="Uninstall fwd successful", 516 onfail="Uninstall fwd failed" ) 517 ''' 518 main.Mininet1.handle.sendline( "py [ h.cmd( \"arping -c 1 10.1.1.1 \" ) for h in net.hosts ] ") 519 import time 520 time.sleep(60) 521 ''' 522 523 main.step( "Check app ids" ) 524 threads = [] 525 appCheck2 = main.TRUE 526 for i in range( main.numCtrls ): 527 t = main.Thread( target=main.CLIs[i].appToIDCheck, 528 name="appToIDCheck-" + str( i ), 529 args=[] ) 530 threads.append( t ) 531 t.start() 532 533 for t in threads: 534 t.join() 535 appCheck2 = appCheck2 and t.result 536 if appCheck2 != main.TRUE: 537 main.log.warn( main.CLIs[0].apps() ) 538 main.log.warn( main.CLIs[0].appIDs() ) 539 utilities.assert_equals( expect=main.TRUE, actual=appCheck2, 540 onpass="App Ids seem to be correct", 541 onfail="Something is wrong with app Ids" ) 542 543 main.step( "Add host intents via cli" ) 544 intentIds = [] 545 # TODO: move the host numbers to params 546 # Maybe look at all the paths we ping? 547 intentAddResult = True 548 hostResult = main.TRUE 549 for i in range( 8, 18 ): 550 main.log.info( "Adding host intent between h" + str( i ) + 551 " and h" + str( i + 10 ) ) 552 host1 = "00:00:00:00:00:" + \ 553 str( hex( i )[ 2: ] ).zfill( 2 ).upper() 554 host2 = "00:00:00:00:00:" + \ 555 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper() 556 # NOTE: getHost can return None 557 host1Dict = main.ONOScli1.getHost( host1 ) 558 host2Dict = main.ONOScli1.getHost( host2 ) 559 host1Id = None 560 host2Id = None 561 if host1Dict and host2Dict: 562 host1Id = host1Dict.get( 'id', None ) 563 host2Id = host2Dict.get( 'id', None ) 564 if host1Id and host2Id: 565 nodeNum = ( i % main.numCtrls ) 566 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id ) 567 if tmpId: 568 main.log.info( "Added intent with id: " + tmpId ) 569 intentIds.append( tmpId ) 570 else: 571 main.log.error( "addHostIntent returned: " + 572 repr( tmpId ) ) 573 else: 574 main.log.error( "Error, getHost() failed for h" + str( i ) + 575 " and/or h" + str( i + 10 ) ) 576 hosts = main.CLIs[ 0 ].hosts() 577 main.log.warn( "Hosts output: " ) 578 try: 579 main.log.warn( json.dumps( json.loads( hosts ), 580 sort_keys=True, 581 indent=4, 582 separators=( ',', ': ' ) ) ) 583 except ( ValueError, TypeError ): 584 main.log.warn( repr( hosts ) ) 585 hostResult = main.FALSE 586 utilities.assert_equals( expect=main.TRUE, actual=hostResult, 587 onpass="Found a host id for each host", 588 onfail="Error looking up host ids" ) 589 590 intentStart = time.time() 591 onosIds = main.ONOScli1.getAllIntentsId() 592 main.log.info( "Submitted intents: " + str( intentIds ) ) 593 main.log.info( "Intents in ONOS: " + str( onosIds ) ) 594 for intent in intentIds: 595 if intent in onosIds: 596 pass # intent submitted is in onos 597 else: 598 intentAddResult = False 599 if intentAddResult: 600 intentStop = time.time() 601 else: 602 intentStop = None 603 # Print the intent states 604 intents = main.ONOScli1.intents() 605 intentStates = [] 606 installedCheck = True 607 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 608 count = 0 609 try: 610 for intent in json.loads( intents ): 611 state = intent.get( 'state', None ) 612 if "INSTALLED" not in state: 613 installedCheck = False 614 intentId = intent.get( 'id', None ) 615 intentStates.append( ( intentId, state ) ) 616 except ( ValueError, TypeError ): 617 main.log.exception( "Error parsing intents" ) 618 # add submitted intents not in the store 619 tmplist = [ i for i, s in intentStates ] 620 missingIntents = False 621 for i in intentIds: 622 if i not in tmplist: 623 intentStates.append( ( i, " - " ) ) 624 missingIntents = True 625 intentStates.sort() 626 for i, s in intentStates: 627 count += 1 628 main.log.info( "%-6s%-15s%-15s" % 629 ( str( count ), str( i ), str( s ) ) ) 630 leaders = main.ONOScli1.leaders() 631 try: 632 missing = False 633 if leaders: 634 parsedLeaders = json.loads( leaders ) 635 main.log.warn( json.dumps( parsedLeaders, 636 sort_keys=True, 637 indent=4, 638 separators=( ',', ': ' ) ) ) 639 # check for all intent partitions 640 topics = [] 641 for i in range( 14 ): 642 topics.append( "intent-partition-" + str( i ) ) 643 main.log.debug( topics ) 644 ONOStopics = [ j['topic'] for j in parsedLeaders ] 645 for topic in topics: 646 if topic not in ONOStopics: 647 main.log.error( "Error: " + topic + 648 " not in leaders" ) 649 missing = True 650 else: 651 main.log.error( "leaders() returned None" ) 652 except ( ValueError, TypeError ): 653 main.log.exception( "Error parsing leaders" ) 654 main.log.error( repr( leaders ) ) 655 # Check all nodes 656 if missing: 657 for node in main.CLIs: 658 response = node.leaders( jsonFormat=False) 659 main.log.warn( str( node.name ) + " leaders output: \n" + 660 str( response ) ) 661 662 partitions = main.ONOScli1.partitions() 663 try: 664 if partitions : 665 parsedPartitions = json.loads( partitions ) 666 main.log.warn( json.dumps( parsedPartitions, 667 sort_keys=True, 668 indent=4, 669 separators=( ',', ': ' ) ) ) 670 # TODO check for a leader in all paritions 671 # TODO check for consistency among nodes 672 else: 673 main.log.error( "partitions() returned None" ) 674 except ( ValueError, TypeError ): 675 main.log.exception( "Error parsing partitions" ) 676 main.log.error( repr( partitions ) ) 677 pendingMap = main.ONOScli1.pendingMap() 678 try: 679 if pendingMap : 680 parsedPending = json.loads( pendingMap ) 681 main.log.warn( json.dumps( parsedPending, 682 sort_keys=True, 683 indent=4, 684 separators=( ',', ': ' ) ) ) 685 # TODO check something here? 686 else: 687 main.log.error( "pendingMap() returned None" ) 688 except ( ValueError, TypeError ): 689 main.log.exception( "Error parsing pending map" ) 690 main.log.error( repr( pendingMap ) ) 691 692 intentAddResult = bool( intentAddResult and not missingIntents and 693 installedCheck ) 694 if not intentAddResult: 695 main.log.error( "Error in pushing host intents to ONOS" ) 696 697 main.step( "Intent Anti-Entropy dispersion" ) 698 for i in range(100): 699 correct = True 700 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) ) 701 for cli in main.CLIs: 702 onosIds = [] 703 ids = cli.getAllIntentsId() 704 onosIds.append( ids ) 705 main.log.debug( "Intents in " + cli.name + ": " + 706 str( sorted( onosIds ) ) ) 707 if sorted( ids ) != sorted( intentIds ): 708 main.log.warn( "Set of intent IDs doesn't match" ) 709 correct = False 710 break 711 else: 712 intents = json.loads( cli.intents() ) 713 for intent in intents: 714 if intent[ 'state' ] != "INSTALLED": 715 main.log.warn( "Intent " + intent[ 'id' ] + 716 " is " + intent[ 'state' ] ) 717 correct = False 718 break 719 if correct: 720 break 721 else: 722 time.sleep(1) 723 if not intentStop: 724 intentStop = time.time() 725 global gossipTime 726 gossipTime = intentStop - intentStart 727 main.log.info( "It took about " + str( gossipTime ) + 728 " seconds for all intents to appear in each node" ) 729 # FIXME: make this time configurable/calculate based off of number of 730 # nodes and gossip rounds 731 utilities.assert_greater_equals( 732 expect=40, actual=gossipTime, 733 onpass="ECM anti-entropy for intents worked within " + 734 "expected time", 735 onfail="Intent ECM anti-entropy took too long" ) 736 if gossipTime <= 40: 737 intentAddResult = True 738 739 if not intentAddResult or "key" in pendingMap: 740 import time 741 installedCheck = True 742 main.log.info( "Sleeping 60 seconds to see if intents are found" ) 743 time.sleep( 60 ) 744 onosIds = main.ONOScli1.getAllIntentsId() 745 main.log.info( "Submitted intents: " + str( intentIds ) ) 746 main.log.info( "Intents in ONOS: " + str( onosIds ) ) 747 # Print the intent states 748 intents = main.ONOScli1.intents() 749 intentStates = [] 750 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 751 count = 0 752 try: 753 for intent in json.loads( intents ): 754 # Iter through intents of a node 755 state = intent.get( 'state', None ) 756 if "INSTALLED" not in state: 757 installedCheck = False 758 intentId = intent.get( 'id', None ) 759 intentStates.append( ( intentId, state ) ) 760 except ( ValueError, TypeError ): 761 main.log.exception( "Error parsing intents" ) 762 # add submitted intents not in the store 763 tmplist = [ i for i, s in intentStates ] 764 for i in intentIds: 765 if i not in tmplist: 766 intentStates.append( ( i, " - " ) ) 767 intentStates.sort() 768 for i, s in intentStates: 769 count += 1 770 main.log.info( "%-6s%-15s%-15s" % 771 ( str( count ), str( i ), str( s ) ) ) 772 leaders = main.ONOScli1.leaders() 773 try: 774 missing = False 775 if leaders: 776 parsedLeaders = json.loads( leaders ) 777 main.log.warn( json.dumps( parsedLeaders, 778 sort_keys=True, 779 indent=4, 780 separators=( ',', ': ' ) ) ) 781 # check for all intent partitions 782 # check for election 783 topics = [] 784 for i in range( 14 ): 785 topics.append( "intent-partition-" + str( i ) ) 786 # FIXME: this should only be after we start the app 787 topics.append( "org.onosproject.election" ) 788 main.log.debug( topics ) 789 ONOStopics = [ j['topic'] for j in parsedLeaders ] 790 for topic in topics: 791 if topic not in ONOStopics: 792 main.log.error( "Error: " + topic + 793 " not in leaders" ) 794 missing = True 795 else: 796 main.log.error( "leaders() returned None" ) 797 except ( ValueError, TypeError ): 798 main.log.exception( "Error parsing leaders" ) 799 main.log.error( repr( leaders ) ) 800 # Check all nodes 801 if missing: 802 for node in main.CLIs: 803 response = node.leaders( jsonFormat=False) 804 main.log.warn( str( node.name ) + " leaders output: \n" + 805 str( response ) ) 806 807 partitions = main.ONOScli1.partitions() 808 try: 809 if partitions : 810 parsedPartitions = json.loads( partitions ) 811 main.log.warn( json.dumps( parsedPartitions, 812 sort_keys=True, 813 indent=4, 814 separators=( ',', ': ' ) ) ) 815 # TODO check for a leader in all paritions 816 # TODO check for consistency among nodes 817 else: 818 main.log.error( "partitions() returned None" ) 819 except ( ValueError, TypeError ): 820 main.log.exception( "Error parsing partitions" ) 821 main.log.error( repr( partitions ) ) 822 pendingMap = main.ONOScli1.pendingMap() 823 try: 824 if pendingMap : 825 parsedPending = json.loads( pendingMap ) 826 main.log.warn( json.dumps( parsedPending, 827 sort_keys=True, 828 indent=4, 829 separators=( ',', ': ' ) ) ) 830 # TODO check something here? 831 else: 832 main.log.error( "pendingMap() returned None" ) 833 except ( ValueError, TypeError ): 834 main.log.exception( "Error parsing pending map" ) 835 main.log.error( repr( pendingMap ) )
836
837 - def CASE4( self, main ):
838 """ 839 Ping across added host intents 840 """ 841 import json 842 import time 843 assert main.numCtrls, "main.numCtrls not defined" 844 assert main, "main not defined" 845 assert utilities.assert_equals, "utilities.assert_equals not defined" 846 assert main.CLIs, "main.CLIs not defined" 847 assert main.nodes, "main.nodes not defined" 848 main.case( "Verify connectivity by sendind traffic across Intents" ) 849 main.caseExplanation = "Ping across added host intents to check " +\ 850 "functionality and check the state of " +\ 851 "the intent" 852 main.step( "Ping across added host intents" ) 853 PingResult = main.TRUE 854 for i in range( 8, 18 ): 855 ping = main.Mininet1.pingHost( src="h" + str( i ), 856 target="h" + str( i + 10 ) ) 857 PingResult = PingResult and ping 858 if ping == main.FALSE: 859 main.log.warn( "Ping failed between h" + str( i ) + 860 " and h" + str( i + 10 ) ) 861 elif ping == main.TRUE: 862 main.log.info( "Ping test passed!" ) 863 # Don't set PingResult or you'd override failures 864 if PingResult == main.FALSE: 865 main.log.error( 866 "Intents have not been installed correctly, pings failed." ) 867 # TODO: pretty print 868 main.log.warn( "ONOS1 intents: " ) 869 try: 870 tmpIntents = main.ONOScli1.intents() 871 main.log.warn( json.dumps( json.loads( tmpIntents ), 872 sort_keys=True, 873 indent=4, 874 separators=( ',', ': ' ) ) ) 875 except ( ValueError, TypeError ): 876 main.log.warn( repr( tmpIntents ) ) 877 utilities.assert_equals( 878 expect=main.TRUE, 879 actual=PingResult, 880 onpass="Intents have been installed correctly and pings work", 881 onfail="Intents have not been installed correctly, pings failed." ) 882 883 main.step( "Check Intent state" ) 884 installedCheck = False 885 loopCount = 0 886 while not installedCheck and loopCount < 40: 887 installedCheck = True 888 # Print the intent states 889 intents = main.ONOScli1.intents() 890 intentStates = [] 891 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 892 count = 0 893 # Iter through intents of a node 894 try: 895 for intent in json.loads( intents ): 896 state = intent.get( 'state', None ) 897 if "INSTALLED" not in state: 898 installedCheck = False 899 intentId = intent.get( 'id', None ) 900 intentStates.append( ( intentId, state ) ) 901 except ( ValueError, TypeError ): 902 main.log.exception( "Error parsing intents." ) 903 # Print states 904 intentStates.sort() 905 for i, s in intentStates: 906 count += 1 907 main.log.info( "%-6s%-15s%-15s" % 908 ( str( count ), str( i ), str( s ) ) ) 909 if not installedCheck: 910 time.sleep( 1 ) 911 loopCount += 1 912 utilities.assert_equals( expect=True, actual=installedCheck, 913 onpass="Intents are all INSTALLED", 914 onfail="Intents are not all in " + 915 "INSTALLED state" ) 916 917 main.step( "Check leadership of topics" ) 918 leaders = main.ONOScli1.leaders() 919 topicCheck = main.TRUE 920 try: 921 if leaders: 922 parsedLeaders = json.loads( leaders ) 923 main.log.warn( json.dumps( parsedLeaders, 924 sort_keys=True, 925 indent=4, 926 separators=( ',', ': ' ) ) ) 927 # check for all intent partitions 928 # check for election 929 # TODO: Look at Devices as topics now that it uses this system 930 topics = [] 931 for i in range( 14 ): 932 topics.append( "intent-partition-" + str( i ) ) 933 # FIXME: this should only be after we start the app 934 # FIXME: topics.append( "org.onosproject.election" ) 935 # Print leaders output 936 main.log.debug( topics ) 937 ONOStopics = [ j['topic'] for j in parsedLeaders ] 938 for topic in topics: 939 if topic not in ONOStopics: 940 main.log.error( "Error: " + topic + 941 " not in leaders" ) 942 topicCheck = main.FALSE 943 else: 944 main.log.error( "leaders() returned None" ) 945 topicCheck = main.FALSE 946 except ( ValueError, TypeError ): 947 topicCheck = main.FALSE 948 main.log.exception( "Error parsing leaders" ) 949 main.log.error( repr( leaders ) ) 950 # TODO: Check for a leader of these topics 951 # Check all nodes 952 if topicCheck: 953 for node in main.CLIs: 954 response = node.leaders( jsonFormat=False) 955 main.log.warn( str( node.name ) + " leaders output: \n" + 956 str( response ) ) 957 958 utilities.assert_equals( expect=main.TRUE, actual=topicCheck, 959 onpass="intent Partitions is in leaders", 960 onfail="Some topics were lost " ) 961 # Print partitions 962 partitions = main.ONOScli1.partitions() 963 try: 964 if partitions : 965 parsedPartitions = json.loads( partitions ) 966 main.log.warn( json.dumps( parsedPartitions, 967 sort_keys=True, 968 indent=4, 969 separators=( ',', ': ' ) ) ) 970 # TODO check for a leader in all paritions 971 # TODO check for consistency among nodes 972 else: 973 main.log.error( "partitions() returned None" ) 974 except ( ValueError, TypeError ): 975 main.log.exception( "Error parsing partitions" ) 976 main.log.error( repr( partitions ) ) 977 # Print Pending Map 978 pendingMap = main.ONOScli1.pendingMap() 979 try: 980 if pendingMap : 981 parsedPending = json.loads( pendingMap ) 982 main.log.warn( json.dumps( parsedPending, 983 sort_keys=True, 984 indent=4, 985 separators=( ',', ': ' ) ) ) 986 # TODO check something here? 987 else: 988 main.log.error( "pendingMap() returned None" ) 989 except ( ValueError, TypeError ): 990 main.log.exception( "Error parsing pending map" ) 991 main.log.error( repr( pendingMap ) ) 992 993 if not installedCheck: 994 main.log.info( "Waiting 60 seconds to see if the state of " + 995 "intents change" ) 996 time.sleep( 60 ) 997 # Print the intent states 998 intents = main.ONOScli1.intents() 999 intentStates = [] 1000 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 1001 count = 0 1002 # Iter through intents of a node 1003 try: 1004 for intent in json.loads( intents ): 1005 state = intent.get( 'state', None ) 1006 if "INSTALLED" not in state: 1007 installedCheck = False 1008 intentId = intent.get( 'id', None ) 1009 intentStates.append( ( intentId, state ) ) 1010 except ( ValueError, TypeError ): 1011 main.log.exception( "Error parsing intents." ) 1012 intentStates.sort() 1013 for i, s in intentStates: 1014 count += 1 1015 main.log.info( "%-6s%-15s%-15s" % 1016 ( str( count ), str( i ), str( s ) ) ) 1017 leaders = main.ONOScli1.leaders() 1018 try: 1019 missing = False 1020 if leaders: 1021 parsedLeaders = json.loads( leaders ) 1022 main.log.warn( json.dumps( parsedLeaders, 1023 sort_keys=True, 1024 indent=4, 1025 separators=( ',', ': ' ) ) ) 1026 # check for all intent partitions 1027 # check for election 1028 topics = [] 1029 for i in range( 14 ): 1030 topics.append( "intent-partition-" + str( i ) ) 1031 # FIXME: this should only be after we start the app 1032 topics.append( "org.onosproject.election" ) 1033 main.log.debug( topics ) 1034 ONOStopics = [ j['topic'] for j in parsedLeaders ] 1035 for topic in topics: 1036 if topic not in ONOStopics: 1037 main.log.error( "Error: " + topic + 1038 " not in leaders" ) 1039 missing = True 1040 else: 1041 main.log.error( "leaders() returned None" ) 1042 except ( ValueError, TypeError ): 1043 main.log.exception( "Error parsing leaders" ) 1044 main.log.error( repr( leaders ) ) 1045 if missing: 1046 for node in main.CLIs: 1047 response = node.leaders( jsonFormat=False) 1048 main.log.warn( str( node.name ) + " leaders output: \n" + 1049 str( response ) ) 1050 1051 partitions = main.ONOScli1.partitions() 1052 try: 1053 if partitions : 1054 parsedPartitions = json.loads( partitions ) 1055 main.log.warn( json.dumps( parsedPartitions, 1056 sort_keys=True, 1057 indent=4, 1058 separators=( ',', ': ' ) ) ) 1059 # TODO check for a leader in all paritions 1060 # TODO check for consistency among nodes 1061 else: 1062 main.log.error( "partitions() returned None" ) 1063 except ( ValueError, TypeError ): 1064 main.log.exception( "Error parsing partitions" ) 1065 main.log.error( repr( partitions ) ) 1066 pendingMap = main.ONOScli1.pendingMap() 1067 try: 1068 if pendingMap : 1069 parsedPending = json.loads( pendingMap ) 1070 main.log.warn( json.dumps( parsedPending, 1071 sort_keys=True, 1072 indent=4, 1073 separators=( ',', ': ' ) ) ) 1074 # TODO check something here? 1075 else: 1076 main.log.error( "pendingMap() returned None" ) 1077 except ( ValueError, TypeError ): 1078 main.log.exception( "Error parsing pending map" ) 1079 main.log.error( repr( pendingMap ) ) 1080 # Print flowrules 1081 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) ) 1082 main.step( "Wait a minute then ping again" ) 1083 # the wait is above 1084 PingResult = main.TRUE 1085 for i in range( 8, 18 ): 1086 ping = main.Mininet1.pingHost( src="h" + str( i ), 1087 target="h" + str( i + 10 ) ) 1088 PingResult = PingResult and ping 1089 if ping == main.FALSE: 1090 main.log.warn( "Ping failed between h" + str( i ) + 1091 " and h" + str( i + 10 ) ) 1092 elif ping == main.TRUE: 1093 main.log.info( "Ping test passed!" ) 1094 # Don't set PingResult or you'd override failures 1095 if PingResult == main.FALSE: 1096 main.log.error( 1097 "Intents have not been installed correctly, pings failed." ) 1098 # TODO: pretty print 1099 main.log.warn( "ONOS1 intents: " ) 1100 try: 1101 tmpIntents = main.ONOScli1.intents() 1102 main.log.warn( json.dumps( json.loads( tmpIntents ), 1103 sort_keys=True, 1104 indent=4, 1105 separators=( ',', ': ' ) ) ) 1106 except ( ValueError, TypeError ): 1107 main.log.warn( repr( tmpIntents ) ) 1108 utilities.assert_equals( 1109 expect=main.TRUE, 1110 actual=PingResult, 1111 onpass="Intents have been installed correctly and pings work", 1112 onfail="Intents have not been installed correctly, pings failed." )
1113
1114 - def CASE5( self, main ):
1115 """ 1116 Reading state of ONOS 1117 """ 1118 import json 1119 import time 1120 assert main.numCtrls, "main.numCtrls not defined" 1121 assert main, "main not defined" 1122 assert utilities.assert_equals, "utilities.assert_equals not defined" 1123 assert main.CLIs, "main.CLIs not defined" 1124 assert main.nodes, "main.nodes not defined" 1125 1126 main.case( "Setting up and gathering data for current state" ) 1127 # The general idea for this test case is to pull the state of 1128 # ( intents,flows, topology,... ) from each ONOS node 1129 # We can then compare them with each other and also with past states 1130 1131 main.step( "Check that each switch has a master" ) 1132 global mastershipState 1133 mastershipState = '[]' 1134 1135 # Assert that each device has a master 1136 rolesNotNull = main.TRUE 1137 threads = [] 1138 for i in range( main.numCtrls ): 1139 t = main.Thread( target=main.CLIs[i].rolesNotNull, 1140 name="rolesNotNull-" + str( i ), 1141 args=[] ) 1142 threads.append( t ) 1143 t.start() 1144 1145 for t in threads: 1146 t.join() 1147 rolesNotNull = rolesNotNull and t.result 1148 utilities.assert_equals( 1149 expect=main.TRUE, 1150 actual=rolesNotNull, 1151 onpass="Each device has a master", 1152 onfail="Some devices don't have a master assigned" ) 1153 1154 main.step( "Get the Mastership of each switch from each controller" ) 1155 ONOSMastership = [] 1156 mastershipCheck = main.FALSE 1157 consistentMastership = True 1158 rolesResults = True 1159 threads = [] 1160 for i in range( main.numCtrls ): 1161 t = main.Thread( target=main.CLIs[i].roles, 1162 name="roles-" + str( i ), 1163 args=[] ) 1164 threads.append( t ) 1165 t.start() 1166 1167 for t in threads: 1168 t.join() 1169 ONOSMastership.append( t.result ) 1170 1171 for i in range( main.numCtrls ): 1172 if not ONOSMastership[i] or "Error" in ONOSMastership[i]: 1173 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1174 " roles" ) 1175 main.log.warn( 1176 "ONOS" + str( i + 1 ) + " mastership response: " + 1177 repr( ONOSMastership[i] ) ) 1178 rolesResults = False 1179 utilities.assert_equals( 1180 expect=True, 1181 actual=rolesResults, 1182 onpass="No error in reading roles output", 1183 onfail="Error in reading roles from ONOS" ) 1184 1185 main.step( "Check for consistency in roles from each controller" ) 1186 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ): 1187 main.log.info( 1188 "Switch roles are consistent across all ONOS nodes" ) 1189 else: 1190 consistentMastership = False 1191 utilities.assert_equals( 1192 expect=True, 1193 actual=consistentMastership, 1194 onpass="Switch roles are consistent across all ONOS nodes", 1195 onfail="ONOS nodes have different views of switch roles" ) 1196 1197 if rolesResults and not consistentMastership: 1198 for i in range( main.numCtrls ): 1199 try: 1200 main.log.warn( 1201 "ONOS" + str( i + 1 ) + " roles: ", 1202 json.dumps( 1203 json.loads( ONOSMastership[ i ] ), 1204 sort_keys=True, 1205 indent=4, 1206 separators=( ',', ': ' ) ) ) 1207 except ( ValueError, TypeError ): 1208 main.log.warn( repr( ONOSMastership[ i ] ) ) 1209 elif rolesResults and consistentMastership: 1210 mastershipCheck = main.TRUE 1211 mastershipState = ONOSMastership[ 0 ] 1212 1213 main.step( "Get the intents from each controller" ) 1214 global intentState 1215 intentState = [] 1216 ONOSIntents = [] 1217 intentCheck = main.FALSE 1218 consistentIntents = True 1219 intentsResults = True 1220 threads = [] 1221 for i in range( main.numCtrls ): 1222 t = main.Thread( target=main.CLIs[i].intents, 1223 name="intents-" + str( i ), 1224 args=[], 1225 kwargs={ 'jsonFormat': True } ) 1226 threads.append( t ) 1227 t.start() 1228 1229 for t in threads: 1230 t.join() 1231 ONOSIntents.append( t.result ) 1232 1233 for i in range( main.numCtrls ): 1234 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]: 1235 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1236 " intents" ) 1237 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " + 1238 repr( ONOSIntents[ i ] ) ) 1239 intentsResults = False 1240 utilities.assert_equals( 1241 expect=True, 1242 actual=intentsResults, 1243 onpass="No error in reading intents output", 1244 onfail="Error in reading intents from ONOS" ) 1245 1246 main.step( "Check for consistency in Intents from each controller" ) 1247 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ): 1248 main.log.info( "Intents are consistent across all ONOS " + 1249 "nodes" ) 1250 else: 1251 consistentIntents = False 1252 main.log.error( "Intents not consistent" ) 1253 utilities.assert_equals( 1254 expect=True, 1255 actual=consistentIntents, 1256 onpass="Intents are consistent across all ONOS nodes", 1257 onfail="ONOS nodes have different views of intents" ) 1258 1259 if intentsResults: 1260 # Try to make it easy to figure out what is happening 1261 # 1262 # Intent ONOS1 ONOS2 ... 1263 # 0x01 INSTALLED INSTALLING 1264 # ... ... ... 1265 # ... ... ... 1266 title = " Id" 1267 for n in range( main.numCtrls ): 1268 title += " " * 10 + "ONOS" + str( n + 1 ) 1269 main.log.warn( title ) 1270 # get all intent keys in the cluster 1271 keys = [] 1272 try: 1273 # Get the set of all intent keys 1274 for nodeStr in ONOSIntents: 1275 node = json.loads( nodeStr ) 1276 for intent in node: 1277 keys.append( intent.get( 'id' ) ) 1278 keys = set( keys ) 1279 # For each intent key, print the state on each node 1280 for key in keys: 1281 row = "%-13s" % key 1282 for nodeStr in ONOSIntents: 1283 node = json.loads( nodeStr ) 1284 for intent in node: 1285 if intent.get( 'id', "Error" ) == key: 1286 row += "%-15s" % intent.get( 'state' ) 1287 main.log.warn( row ) 1288 # End of intent state table 1289 except ValueError as e: 1290 main.log.exception( e ) 1291 main.log.debug( "nodeStr was: " + repr( nodeStr ) ) 1292 1293 if intentsResults and not consistentIntents: 1294 # print the json objects 1295 n = len(ONOSIntents) 1296 main.log.debug( "ONOS" + str( n ) + " intents: " ) 1297 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ), 1298 sort_keys=True, 1299 indent=4, 1300 separators=( ',', ': ' ) ) ) 1301 for i in range( main.numCtrls ): 1302 if ONOSIntents[ i ] != ONOSIntents[ -1 ]: 1303 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " ) 1304 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ), 1305 sort_keys=True, 1306 indent=4, 1307 separators=( ',', ': ' ) ) ) 1308 else: 1309 main.log.debug( main.nodes[ i ].name + " intents match ONOS" + 1310 str( n ) + " intents" ) 1311 elif intentsResults and consistentIntents: 1312 intentCheck = main.TRUE 1313 intentState = ONOSIntents[ 0 ] 1314 1315 main.step( "Get the flows from each controller" ) 1316 global flowState 1317 flowState = [] 1318 ONOSFlows = [] 1319 ONOSFlowsJson = [] 1320 flowCheck = main.FALSE 1321 consistentFlows = True 1322 flowsResults = True 1323 threads = [] 1324 for i in range( main.numCtrls ): 1325 t = main.Thread( target=main.CLIs[i].flows, 1326 name="flows-" + str( i ), 1327 args=[], 1328 kwargs={ 'jsonFormat': True } ) 1329 threads.append( t ) 1330 t.start() 1331 1332 # NOTE: Flows command can take some time to run 1333 time.sleep(30) 1334 for t in threads: 1335 t.join() 1336 result = t.result 1337 ONOSFlows.append( result ) 1338 1339 for i in range( main.numCtrls ): 1340 num = str( i + 1 ) 1341 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]: 1342 main.log.error( "Error in getting ONOS" + num + " flows" ) 1343 main.log.warn( "ONOS" + num + " flows response: " + 1344 repr( ONOSFlows[ i ] ) ) 1345 flowsResults = False 1346 ONOSFlowsJson.append( None ) 1347 else: 1348 try: 1349 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) ) 1350 except ( ValueError, TypeError ): 1351 # FIXME: change this to log.error? 1352 main.log.exception( "Error in parsing ONOS" + num + 1353 " response as json." ) 1354 main.log.error( repr( ONOSFlows[ i ] ) ) 1355 ONOSFlowsJson.append( None ) 1356 flowsResults = False 1357 utilities.assert_equals( 1358 expect=True, 1359 actual=flowsResults, 1360 onpass="No error in reading flows output", 1361 onfail="Error in reading flows from ONOS" ) 1362 1363 main.step( "Check for consistency in Flows from each controller" ) 1364 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ] 1365 if all( tmp ): 1366 main.log.info( "Flow count is consistent across all ONOS nodes" ) 1367 else: 1368 consistentFlows = False 1369 utilities.assert_equals( 1370 expect=True, 1371 actual=consistentFlows, 1372 onpass="The flow count is consistent across all ONOS nodes", 1373 onfail="ONOS nodes have different flow counts" ) 1374 1375 if flowsResults and not consistentFlows: 1376 for i in range( main.numCtrls ): 1377 try: 1378 main.log.warn( 1379 "ONOS" + str( i + 1 ) + " flows: " + 1380 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True, 1381 indent=4, separators=( ',', ': ' ) ) ) 1382 except ( ValueError, TypeError ): 1383 main.log.warn( 1384 "ONOS" + str( i + 1 ) + " flows: " + 1385 repr( ONOSFlows[ i ] ) ) 1386 elif flowsResults and consistentFlows: 1387 flowCheck = main.TRUE 1388 flowState = ONOSFlows[ 0 ] 1389 1390 main.step( "Get the OF Table entries" ) 1391 global flows 1392 flows = [] 1393 for i in range( 1, 29 ): 1394 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) ) 1395 if flowCheck == main.FALSE: 1396 for table in flows: 1397 main.log.warn( table ) 1398 # TODO: Compare switch flow tables with ONOS flow tables 1399 1400 main.step( "Start continuous pings" ) 1401 main.Mininet2.pingLong( 1402 src=main.params[ 'PING' ][ 'source1' ], 1403 target=main.params[ 'PING' ][ 'target1' ], 1404 pingTime=500 ) 1405 main.Mininet2.pingLong( 1406 src=main.params[ 'PING' ][ 'source2' ], 1407 target=main.params[ 'PING' ][ 'target2' ], 1408 pingTime=500 ) 1409 main.Mininet2.pingLong( 1410 src=main.params[ 'PING' ][ 'source3' ], 1411 target=main.params[ 'PING' ][ 'target3' ], 1412 pingTime=500 ) 1413 main.Mininet2.pingLong( 1414 src=main.params[ 'PING' ][ 'source4' ], 1415 target=main.params[ 'PING' ][ 'target4' ], 1416 pingTime=500 ) 1417 main.Mininet2.pingLong( 1418 src=main.params[ 'PING' ][ 'source5' ], 1419 target=main.params[ 'PING' ][ 'target5' ], 1420 pingTime=500 ) 1421 main.Mininet2.pingLong( 1422 src=main.params[ 'PING' ][ 'source6' ], 1423 target=main.params[ 'PING' ][ 'target6' ], 1424 pingTime=500 ) 1425 main.Mininet2.pingLong( 1426 src=main.params[ 'PING' ][ 'source7' ], 1427 target=main.params[ 'PING' ][ 'target7' ], 1428 pingTime=500 ) 1429 main.Mininet2.pingLong( 1430 src=main.params[ 'PING' ][ 'source8' ], 1431 target=main.params[ 'PING' ][ 'target8' ], 1432 pingTime=500 ) 1433 main.Mininet2.pingLong( 1434 src=main.params[ 'PING' ][ 'source9' ], 1435 target=main.params[ 'PING' ][ 'target9' ], 1436 pingTime=500 ) 1437 main.Mininet2.pingLong( 1438 src=main.params[ 'PING' ][ 'source10' ], 1439 target=main.params[ 'PING' ][ 'target10' ], 1440 pingTime=500 ) 1441 1442 main.step( "Collecting topology information from ONOS" ) 1443 devices = [] 1444 threads = [] 1445 for i in range( main.numCtrls ): 1446 t = main.Thread( target=main.CLIs[i].devices, 1447 name="devices-" + str( i ), 1448 args=[ ] ) 1449 threads.append( t ) 1450 t.start() 1451 1452 for t in threads: 1453 t.join() 1454 devices.append( t.result ) 1455 hosts = [] 1456 threads = [] 1457 for i in range( main.numCtrls ): 1458 t = main.Thread( target=main.CLIs[i].hosts, 1459 name="hosts-" + str( i ), 1460 args=[ ] ) 1461 threads.append( t ) 1462 t.start() 1463 1464 for t in threads: 1465 t.join() 1466 try: 1467 hosts.append( json.loads( t.result ) ) 1468 except ( ValueError, TypeError ): 1469 # FIXME: better handling of this, print which node 1470 # Maybe use thread name? 1471 main.log.exception( "Error parsing json output of hosts" ) 1472 # FIXME: should this be an empty json object instead? 1473 hosts.append( None ) 1474 1475 ports = [] 1476 threads = [] 1477 for i in range( main.numCtrls ): 1478 t = main.Thread( target=main.CLIs[i].ports, 1479 name="ports-" + str( i ), 1480 args=[ ] ) 1481 threads.append( t ) 1482 t.start() 1483 1484 for t in threads: 1485 t.join() 1486 ports.append( t.result ) 1487 links = [] 1488 threads = [] 1489 for i in range( main.numCtrls ): 1490 t = main.Thread( target=main.CLIs[i].links, 1491 name="links-" + str( i ), 1492 args=[ ] ) 1493 threads.append( t ) 1494 t.start() 1495 1496 for t in threads: 1497 t.join() 1498 links.append( t.result ) 1499 clusters = [] 1500 threads = [] 1501 for i in range( main.numCtrls ): 1502 t = main.Thread( target=main.CLIs[i].clusters, 1503 name="clusters-" + str( i ), 1504 args=[ ] ) 1505 threads.append( t ) 1506 t.start() 1507 1508 for t in threads: 1509 t.join() 1510 clusters.append( t.result ) 1511 # Compare json objects for hosts and dataplane clusters 1512 1513 # hosts 1514 main.step( "Host view is consistent across ONOS nodes" ) 1515 consistentHostsResult = main.TRUE 1516 for controller in range( len( hosts ) ): 1517 controllerStr = str( controller + 1 ) 1518 if "Error" not in hosts[ controller ]: 1519 if hosts[ controller ] == hosts[ 0 ]: 1520 continue 1521 else: # hosts not consistent 1522 main.log.error( "hosts from ONOS" + 1523 controllerStr + 1524 " is inconsistent with ONOS1" ) 1525 main.log.warn( repr( hosts[ controller ] ) ) 1526 consistentHostsResult = main.FALSE 1527 1528 else: 1529 main.log.error( "Error in getting ONOS hosts from ONOS" + 1530 controllerStr ) 1531 consistentHostsResult = main.FALSE 1532 main.log.warn( "ONOS" + controllerStr + 1533 " hosts response: " + 1534 repr( hosts[ controller ] ) ) 1535 utilities.assert_equals( 1536 expect=main.TRUE, 1537 actual=consistentHostsResult, 1538 onpass="Hosts view is consistent across all ONOS nodes", 1539 onfail="ONOS nodes have different views of hosts" ) 1540 1541 main.step( "Each host has an IP address" ) 1542 ipResult = main.TRUE 1543 for controller in range( 0, len( hosts ) ): 1544 controllerStr = str( controller + 1 ) 1545 for host in hosts[ controller ]: 1546 if not host.get( 'ipAddresses', [ ] ): 1547 main.log.error( "DEBUG:Error with host ips on controller" + 1548 controllerStr + ": " + str( host ) ) 1549 ipResult = main.FALSE 1550 utilities.assert_equals( 1551 expect=main.TRUE, 1552 actual=ipResult, 1553 onpass="The ips of the hosts aren't empty", 1554 onfail="The ip of at least one host is missing" ) 1555 1556 # Strongly connected clusters of devices 1557 main.step( "Cluster view is consistent across ONOS nodes" ) 1558 consistentClustersResult = main.TRUE 1559 for controller in range( len( clusters ) ): 1560 controllerStr = str( controller + 1 ) 1561 if "Error" not in clusters[ controller ]: 1562 if clusters[ controller ] == clusters[ 0 ]: 1563 continue 1564 else: # clusters not consistent 1565 main.log.error( "clusters from ONOS" + controllerStr + 1566 " is inconsistent with ONOS1" ) 1567 consistentClustersResult = main.FALSE 1568 1569 else: 1570 main.log.error( "Error in getting dataplane clusters " + 1571 "from ONOS" + controllerStr ) 1572 consistentClustersResult = main.FALSE 1573 main.log.warn( "ONOS" + controllerStr + 1574 " clusters response: " + 1575 repr( clusters[ controller ] ) ) 1576 utilities.assert_equals( 1577 expect=main.TRUE, 1578 actual=consistentClustersResult, 1579 onpass="Clusters view is consistent across all ONOS nodes", 1580 onfail="ONOS nodes have different views of clusters" ) 1581 # there should always only be one cluster 1582 main.step( "Cluster view correct across ONOS nodes" ) 1583 try: 1584 numClusters = len( json.loads( clusters[ 0 ] ) ) 1585 except ( ValueError, TypeError ): 1586 main.log.exception( "Error parsing clusters[0]: " + 1587 repr( clusters[ 0 ] ) ) 1588 clusterResults = main.FALSE 1589 if numClusters == 1: 1590 clusterResults = main.TRUE 1591 utilities.assert_equals( 1592 expect=1, 1593 actual=numClusters, 1594 onpass="ONOS shows 1 SCC", 1595 onfail="ONOS shows " + str( numClusters ) + " SCCs" ) 1596 1597 main.step( "Comparing ONOS topology to MN" ) 1598 devicesResults = main.TRUE 1599 linksResults = main.TRUE 1600 hostsResults = main.TRUE 1601 mnSwitches = main.Mininet1.getSwitches() 1602 mnLinks = main.Mininet1.getLinks() 1603 mnHosts = main.Mininet1.getHosts() 1604 for controller in range( main.numCtrls ): 1605 controllerStr = str( controller + 1 ) 1606 if devices[ controller ] and ports[ controller ] and\ 1607 "Error" not in devices[ controller ] and\ 1608 "Error" not in ports[ controller ]: 1609 1610 currentDevicesResult = main.Mininet1.compareSwitches( 1611 mnSwitches, 1612 json.loads( devices[ controller ] ), 1613 json.loads( ports[ controller ] ) ) 1614 else: 1615 currentDevicesResult = main.FALSE 1616 utilities.assert_equals( expect=main.TRUE, 1617 actual=currentDevicesResult, 1618 onpass="ONOS" + controllerStr + 1619 " Switches view is correct", 1620 onfail="ONOS" + controllerStr + 1621 " Switches view is incorrect" ) 1622 if links[ controller ] and "Error" not in links[ controller ]: 1623 currentLinksResult = main.Mininet1.compareLinks( 1624 mnSwitches, mnLinks, 1625 json.loads( links[ controller ] ) ) 1626 else: 1627 currentLinksResult = main.FALSE 1628 utilities.assert_equals( expect=main.TRUE, 1629 actual=currentLinksResult, 1630 onpass="ONOS" + controllerStr + 1631 " links view is correct", 1632 onfail="ONOS" + controllerStr + 1633 " links view is incorrect" ) 1634 1635 if hosts[ controller ] or "Error" not in hosts[ controller ]: 1636 currentHostsResult = main.Mininet1.compareHosts( 1637 mnHosts, 1638 hosts[ controller ] ) 1639 else: 1640 currentHostsResult = main.FALSE 1641 utilities.assert_equals( expect=main.TRUE, 1642 actual=currentHostsResult, 1643 onpass="ONOS" + controllerStr + 1644 " hosts exist in Mininet", 1645 onfail="ONOS" + controllerStr + 1646 " hosts don't match Mininet" ) 1647 1648 devicesResults = devicesResults and currentDevicesResult 1649 linksResults = linksResults and currentLinksResult 1650 hostsResults = hostsResults and currentHostsResult 1651 1652 main.step( "Device information is correct" ) 1653 utilities.assert_equals( 1654 expect=main.TRUE, 1655 actual=devicesResults, 1656 onpass="Device information is correct", 1657 onfail="Device information is incorrect" ) 1658 1659 main.step( "Links are correct" ) 1660 utilities.assert_equals( 1661 expect=main.TRUE, 1662 actual=linksResults, 1663 onpass="Link are correct", 1664 onfail="Links are incorrect" ) 1665 1666 main.step( "Hosts are correct" ) 1667 utilities.assert_equals( 1668 expect=main.TRUE, 1669 actual=hostsResults, 1670 onpass="Hosts are correct", 1671 onfail="Hosts are incorrect" )
1672
1673 - def CASE6( self, main ):
1674 """ 1675 The Failure case. Since this is the Sanity test, we do nothing. 1676 """ 1677 import time 1678 assert main.numCtrls, "main.numCtrls not defined" 1679 assert main, "main not defined" 1680 assert utilities.assert_equals, "utilities.assert_equals not defined" 1681 assert main.CLIs, "main.CLIs not defined" 1682 assert main.nodes, "main.nodes not defined" 1683 main.case( "Wait 60 seconds instead of inducing a failure" ) 1684 time.sleep( 60 ) 1685 utilities.assert_equals( 1686 expect=main.TRUE, 1687 actual=main.TRUE, 1688 onpass="Sleeping 60 seconds", 1689 onfail="Something is terribly wrong with my math" )
1690
1691 - def CASE7( self, main ):
1692 """ 1693 Check state after ONOS failure 1694 """ 1695 import json 1696 assert main.numCtrls, "main.numCtrls not defined" 1697 assert main, "main not defined" 1698 assert utilities.assert_equals, "utilities.assert_equals not defined" 1699 assert main.CLIs, "main.CLIs not defined" 1700 assert main.nodes, "main.nodes not defined" 1701 main.case( "Running ONOS Constant State Tests" ) 1702 1703 main.step( "Check that each switch has a master" ) 1704 # Assert that each device has a master 1705 rolesNotNull = main.TRUE 1706 threads = [] 1707 for i in range( main.numCtrls ): 1708 t = main.Thread( target=main.CLIs[i].rolesNotNull, 1709 name="rolesNotNull-" + str( i ), 1710 args=[ ] ) 1711 threads.append( t ) 1712 t.start() 1713 1714 for t in threads: 1715 t.join() 1716 rolesNotNull = rolesNotNull and t.result 1717 utilities.assert_equals( 1718 expect=main.TRUE, 1719 actual=rolesNotNull, 1720 onpass="Each device has a master", 1721 onfail="Some devices don't have a master assigned" ) 1722 1723 main.step( "Read device roles from ONOS" ) 1724 ONOSMastership = [] 1725 mastershipCheck = main.FALSE 1726 consistentMastership = True 1727 rolesResults = True 1728 threads = [] 1729 for i in range( main.numCtrls ): 1730 t = main.Thread( target=main.CLIs[i].roles, 1731 name="roles-" + str( i ), 1732 args=[] ) 1733 threads.append( t ) 1734 t.start() 1735 1736 for t in threads: 1737 t.join() 1738 ONOSMastership.append( t.result ) 1739 1740 for i in range( main.numCtrls ): 1741 if not ONOSMastership[i] or "Error" in ONOSMastership[i]: 1742 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1743 " roles" ) 1744 main.log.warn( 1745 "ONOS" + str( i + 1 ) + " mastership response: " + 1746 repr( ONOSMastership[i] ) ) 1747 rolesResults = False 1748 utilities.assert_equals( 1749 expect=True, 1750 actual=rolesResults, 1751 onpass="No error in reading roles output", 1752 onfail="Error in reading roles from ONOS" ) 1753 1754 main.step( "Check for consistency in roles from each controller" ) 1755 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ): 1756 main.log.info( 1757 "Switch roles are consistent across all ONOS nodes" ) 1758 else: 1759 consistentMastership = False 1760 utilities.assert_equals( 1761 expect=True, 1762 actual=consistentMastership, 1763 onpass="Switch roles are consistent across all ONOS nodes", 1764 onfail="ONOS nodes have different views of switch roles" ) 1765 1766 if rolesResults and not consistentMastership: 1767 for i in range( main.numCtrls ): 1768 main.log.warn( 1769 "ONOS" + str( i + 1 ) + " roles: ", 1770 json.dumps( 1771 json.loads( ONOSMastership[ i ] ), 1772 sort_keys=True, 1773 indent=4, 1774 separators=( ',', ': ' ) ) ) 1775 elif rolesResults and not consistentMastership: 1776 mastershipCheck = main.TRUE 1777 1778 description2 = "Compare switch roles from before failure" 1779 main.step( description2 ) 1780 try: 1781 currentJson = json.loads( ONOSMastership[0] ) 1782 oldJson = json.loads( mastershipState ) 1783 except ( ValueError, TypeError ): 1784 main.log.exception( "Something is wrong with parsing " + 1785 "ONOSMastership[0] or mastershipState" ) 1786 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) ) 1787 main.log.error( "mastershipState" + repr( mastershipState ) ) 1788 main.cleanup() 1789 main.exit() 1790 mastershipCheck = main.TRUE 1791 for i in range( 1, 29 ): 1792 switchDPID = str( 1793 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) ) 1794 current = [ switch[ 'master' ] for switch in currentJson 1795 if switchDPID in switch[ 'id' ] ] 1796 old = [ switch[ 'master' ] for switch in oldJson 1797 if switchDPID in switch[ 'id' ] ] 1798 if current == old: 1799 mastershipCheck = mastershipCheck and main.TRUE 1800 else: 1801 main.log.warn( "Mastership of switch %s changed" % switchDPID ) 1802 mastershipCheck = main.FALSE 1803 utilities.assert_equals( 1804 expect=main.TRUE, 1805 actual=mastershipCheck, 1806 onpass="Mastership of Switches was not changed", 1807 onfail="Mastership of some switches changed" ) 1808 mastershipCheck = mastershipCheck and consistentMastership 1809 1810 main.step( "Get the intents and compare across all nodes" ) 1811 ONOSIntents = [] 1812 intentCheck = main.FALSE 1813 consistentIntents = True 1814 intentsResults = True 1815 threads = [] 1816 for i in range( main.numCtrls ): 1817 t = main.Thread( target=main.CLIs[i].intents, 1818 name="intents-" + str( i ), 1819 args=[], 1820 kwargs={ 'jsonFormat': True } ) 1821 threads.append( t ) 1822 t.start() 1823 1824 for t in threads: 1825 t.join() 1826 ONOSIntents.append( t.result ) 1827 1828 for i in range( main.numCtrls ): 1829 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]: 1830 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1831 " intents" ) 1832 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " + 1833 repr( ONOSIntents[ i ] ) ) 1834 intentsResults = False 1835 utilities.assert_equals( 1836 expect=True, 1837 actual=intentsResults, 1838 onpass="No error in reading intents output", 1839 onfail="Error in reading intents from ONOS" ) 1840 1841 main.step( "Check for consistency in Intents from each controller" ) 1842 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ): 1843 main.log.info( "Intents are consistent across all ONOS " + 1844 "nodes" ) 1845 else: 1846 consistentIntents = False 1847 1848 # Try to make it easy to figure out what is happening 1849 # 1850 # Intent ONOS1 ONOS2 ... 1851 # 0x01 INSTALLED INSTALLING 1852 # ... ... ... 1853 # ... ... ... 1854 title = " ID" 1855 for n in range( main.numCtrls ): 1856 title += " " * 10 + "ONOS" + str( n + 1 ) 1857 main.log.warn( title ) 1858 # get all intent keys in the cluster 1859 keys = [] 1860 for nodeStr in ONOSIntents: 1861 node = json.loads( nodeStr ) 1862 for intent in node: 1863 keys.append( intent.get( 'id' ) ) 1864 keys = set( keys ) 1865 for key in keys: 1866 row = "%-13s" % key 1867 for nodeStr in ONOSIntents: 1868 node = json.loads( nodeStr ) 1869 for intent in node: 1870 if intent.get( 'id' ) == key: 1871 row += "%-15s" % intent.get( 'state' ) 1872 main.log.warn( row ) 1873 # End table view 1874 1875 utilities.assert_equals( 1876 expect=True, 1877 actual=consistentIntents, 1878 onpass="Intents are consistent across all ONOS nodes", 1879 onfail="ONOS nodes have different views of intents" ) 1880 intentStates = [] 1881 for node in ONOSIntents: # Iter through ONOS nodes 1882 nodeStates = [] 1883 # Iter through intents of a node 1884 try: 1885 for intent in json.loads( node ): 1886 nodeStates.append( intent[ 'state' ] ) 1887 except ( ValueError, TypeError ): 1888 main.log.exception( "Error in parsing intents" ) 1889 main.log.error( repr( node ) ) 1890 intentStates.append( nodeStates ) 1891 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ] 1892 main.log.info( dict( out ) ) 1893 1894 if intentsResults and not consistentIntents: 1895 for i in range( main.numCtrls ): 1896 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " ) 1897 main.log.warn( json.dumps( 1898 json.loads( ONOSIntents[ i ] ), 1899 sort_keys=True, 1900 indent=4, 1901 separators=( ',', ': ' ) ) ) 1902 elif intentsResults and consistentIntents: 1903 intentCheck = main.TRUE 1904 1905 # NOTE: Store has no durability, so intents are lost across system 1906 # restarts 1907 main.step( "Compare current intents with intents before the failure" ) 1908 # NOTE: this requires case 5 to pass for intentState to be set. 1909 # maybe we should stop the test if that fails? 1910 sameIntents = main.FALSE 1911 if intentState and intentState == ONOSIntents[ 0 ]: 1912 sameIntents = main.TRUE 1913 main.log.info( "Intents are consistent with before failure" ) 1914 # TODO: possibly the states have changed? we may need to figure out 1915 # what the acceptable states are 1916 elif len( intentState ) == len( ONOSIntents[ 0 ] ): 1917 sameIntents = main.TRUE 1918 try: 1919 before = json.loads( intentState ) 1920 after = json.loads( ONOSIntents[ 0 ] ) 1921 for intent in before: 1922 if intent not in after: 1923 sameIntents = main.FALSE 1924 main.log.debug( "Intent is not currently in ONOS " + 1925 "(at least in the same form):" ) 1926 main.log.debug( json.dumps( intent ) ) 1927 except ( ValueError, TypeError ): 1928 main.log.exception( "Exception printing intents" ) 1929 main.log.debug( repr( ONOSIntents[0] ) ) 1930 main.log.debug( repr( intentState ) ) 1931 if sameIntents == main.FALSE: 1932 try: 1933 main.log.debug( "ONOS intents before: " ) 1934 main.log.debug( json.dumps( json.loads( intentState ), 1935 sort_keys=True, indent=4, 1936 separators=( ',', ': ' ) ) ) 1937 main.log.debug( "Current ONOS intents: " ) 1938 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ), 1939 sort_keys=True, indent=4, 1940 separators=( ',', ': ' ) ) ) 1941 except ( ValueError, TypeError ): 1942 main.log.exception( "Exception printing intents" ) 1943 main.log.debug( repr( ONOSIntents[0] ) ) 1944 main.log.debug( repr( intentState ) ) 1945 utilities.assert_equals( 1946 expect=main.TRUE, 1947 actual=sameIntents, 1948 onpass="Intents are consistent with before failure", 1949 onfail="The Intents changed during failure" ) 1950 intentCheck = intentCheck and sameIntents 1951 1952 main.step( "Get the OF Table entries and compare to before " + 1953 "component failure" ) 1954 FlowTables = main.TRUE 1955 flows2 = [] 1956 for i in range( 28 ): 1957 main.log.info( "Checking flow table on s" + str( i + 1 ) ) 1958 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) ) 1959 flows2.append( tmpFlows ) 1960 tempResult = main.Mininet1.flowComp( 1961 flow1=flows[ i ], 1962 flow2=tmpFlows ) 1963 FlowTables = FlowTables and tempResult 1964 if FlowTables == main.FALSE: 1965 main.log.info( "Differences in flow table for switch: s" + 1966 str( i + 1 ) ) 1967 utilities.assert_equals( 1968 expect=main.TRUE, 1969 actual=FlowTables, 1970 onpass="No changes were found in the flow tables", 1971 onfail="Changes were found in the flow tables" ) 1972 1973 main.Mininet2.pingLongKill() 1974 ''' 1975 main.step( "Check the continuous pings to ensure that no packets " + 1976 "were dropped during component failure" ) 1977 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ], 1978 main.params[ 'TESTONIP' ] ) 1979 LossInPings = main.FALSE 1980 # NOTE: checkForLoss returns main.FALSE with 0% packet loss 1981 for i in range( 8, 18 ): 1982 main.log.info( 1983 "Checking for a loss in pings along flow from s" + 1984 str( i ) ) 1985 LossInPings = main.Mininet2.checkForLoss( 1986 "/tmp/ping.h" + 1987 str( i ) ) or LossInPings 1988 if LossInPings == main.TRUE: 1989 main.log.info( "Loss in ping detected" ) 1990 elif LossInPings == main.ERROR: 1991 main.log.info( "There are multiple mininet process running" ) 1992 elif LossInPings == main.FALSE: 1993 main.log.info( "No Loss in the pings" ) 1994 main.log.info( "No loss of dataplane connectivity" ) 1995 utilities.assert_equals( 1996 expect=main.FALSE, 1997 actual=LossInPings, 1998 onpass="No Loss of connectivity", 1999 onfail="Loss of dataplane connectivity detected" ) 2000 ''' 2001 2002 main.step( "Leadership Election is still functional" ) 2003 # Test of LeadershipElection 2004 # NOTE: this only works for the sanity test. In case of failures, 2005 # leader will likely change 2006 leader = main.nodes[ 0 ].ip_address 2007 leaderResult = main.TRUE 2008 for cli in main.CLIs: 2009 leaderN = cli.electionTestLeader() 2010 # verify leader is ONOS1 2011 if leaderN == leader: 2012 # all is well 2013 # NOTE: In failure scenario, this could be a new node, maybe 2014 # check != ONOS1 2015 pass 2016 elif leaderN == main.FALSE: 2017 # error in response 2018 main.log.error( "Something is wrong with " + 2019 "electionTestLeader function, check the" + 2020 " error logs" ) 2021 leaderResult = main.FALSE 2022 elif leader != leaderN: 2023 leaderResult = main.FALSE 2024 main.log.error( cli.name + " sees " + str( leaderN ) + 2025 " as the leader of the election app. " + 2026 "Leader should be " + str( leader ) ) 2027 utilities.assert_equals( 2028 expect=main.TRUE, 2029 actual=leaderResult, 2030 onpass="Leadership election passed", 2031 onfail="Something went wrong with Leadership election" )
2032
2033 - def CASE8( self, main ):
2034 """ 2035 Compare topo 2036 """ 2037 import json 2038 import time 2039 assert main.numCtrls, "main.numCtrls not defined" 2040 assert main, "main not defined" 2041 assert utilities.assert_equals, "utilities.assert_equals not defined" 2042 assert main.CLIs, "main.CLIs not defined" 2043 assert main.nodes, "main.nodes not defined" 2044 2045 main.case( "Compare ONOS Topology view to Mininet topology" ) 2046 main.caseExplanation = "Compare topology objects between Mininet" +\ 2047 " and ONOS" 2048 2049 main.step( "Comparing ONOS topology to MN" ) 2050 devicesResults = main.TRUE 2051 linksResults = main.TRUE 2052 hostsResults = main.TRUE 2053 hostAttachmentResults = True 2054 topoResult = main.FALSE 2055 elapsed = 0 2056 count = 0 2057 main.step( "Collecting topology information from ONOS" ) 2058 startTime = time.time() 2059 # Give time for Gossip to work 2060 while topoResult == main.FALSE and elapsed < 60: 2061 count += 1 2062 cliStart = time.time() 2063 devices = [] 2064 threads = [] 2065 for i in range( main.numCtrls ): 2066 t = main.Thread( target=main.CLIs[i].devices, 2067 name="devices-" + str( i ), 2068 args=[ ] ) 2069 threads.append( t ) 2070 t.start() 2071 2072 for t in threads: 2073 t.join() 2074 devices.append( t.result ) 2075 hosts = [] 2076 ipResult = main.TRUE 2077 threads = [] 2078 for i in range( main.numCtrls ): 2079 t = main.Thread( target=main.CLIs[i].hosts, 2080 name="hosts-" + str( i ), 2081 args=[ ] ) 2082 threads.append( t ) 2083 t.start() 2084 2085 for t in threads: 2086 t.join() 2087 try: 2088 hosts.append( json.loads( t.result ) ) 2089 except ( ValueError, TypeError ): 2090 main.log.exception( "Error parsing hosts results" ) 2091 main.log.error( repr( t.result ) ) 2092 for controller in range( 0, len( hosts ) ): 2093 controllerStr = str( controller + 1 ) 2094 for host in hosts[ controller ]: 2095 if host is None or host.get( 'ipAddresses', [] ) == []: 2096 main.log.error( 2097 "DEBUG:Error with host ipAddresses on controller" + 2098 controllerStr + ": " + str( host ) ) 2099 ipResult = main.FALSE 2100 ports = [] 2101 threads = [] 2102 for i in range( main.numCtrls ): 2103 t = main.Thread( target=main.CLIs[i].ports, 2104 name="ports-" + str( i ), 2105 args=[ ] ) 2106 threads.append( t ) 2107 t.start() 2108 2109 for t in threads: 2110 t.join() 2111 ports.append( t.result ) 2112 links = [] 2113 threads = [] 2114 for i in range( main.numCtrls ): 2115 t = main.Thread( target=main.CLIs[i].links, 2116 name="links-" + str( i ), 2117 args=[ ] ) 2118 threads.append( t ) 2119 t.start() 2120 2121 for t in threads: 2122 t.join() 2123 links.append( t.result ) 2124 clusters = [] 2125 threads = [] 2126 for i in range( main.numCtrls ): 2127 t = main.Thread( target=main.CLIs[i].clusters, 2128 name="clusters-" + str( i ), 2129 args=[ ] ) 2130 threads.append( t ) 2131 t.start() 2132 2133 for t in threads: 2134 t.join() 2135 clusters.append( t.result ) 2136 2137 elapsed = time.time() - startTime 2138 cliTime = time.time() - cliStart 2139 print "Elapsed time: " + str( elapsed ) 2140 print "CLI time: " + str( cliTime ) 2141 2142 mnSwitches = main.Mininet1.getSwitches() 2143 mnLinks = main.Mininet1.getLinks() 2144 mnHosts = main.Mininet1.getHosts() 2145 for controller in range( main.numCtrls ): 2146 controllerStr = str( controller + 1 ) 2147 if devices[ controller ] and ports[ controller ] and\ 2148 "Error" not in devices[ controller ] and\ 2149 "Error" not in ports[ controller ]: 2150 2151 currentDevicesResult = main.Mininet1.compareSwitches( 2152 mnSwitches, 2153 json.loads( devices[ controller ] ), 2154 json.loads( ports[ controller ] ) ) 2155 else: 2156 currentDevicesResult = main.FALSE 2157 utilities.assert_equals( expect=main.TRUE, 2158 actual=currentDevicesResult, 2159 onpass="ONOS" + controllerStr + 2160 " Switches view is correct", 2161 onfail="ONOS" + controllerStr + 2162 " Switches view is incorrect" ) 2163 2164 if links[ controller ] and "Error" not in links[ controller ]: 2165 currentLinksResult = main.Mininet1.compareLinks( 2166 mnSwitches, mnLinks, 2167 json.loads( links[ controller ] ) ) 2168 else: 2169 currentLinksResult = main.FALSE 2170 utilities.assert_equals( expect=main.TRUE, 2171 actual=currentLinksResult, 2172 onpass="ONOS" + controllerStr + 2173 " links view is correct", 2174 onfail="ONOS" + controllerStr + 2175 " links view is incorrect" ) 2176 2177 if hosts[ controller ] or "Error" not in hosts[ controller ]: 2178 currentHostsResult = main.Mininet1.compareHosts( 2179 mnHosts, 2180 hosts[ controller ] ) 2181 else: 2182 currentHostsResult = main.FALSE 2183 utilities.assert_equals( expect=main.TRUE, 2184 actual=currentHostsResult, 2185 onpass="ONOS" + controllerStr + 2186 " hosts exist in Mininet", 2187 onfail="ONOS" + controllerStr + 2188 " hosts don't match Mininet" ) 2189 # CHECKING HOST ATTACHMENT POINTS 2190 hostAttachment = True 2191 zeroHosts = False 2192 # FIXME: topo-HA/obelisk specific mappings: 2193 # key is mac and value is dpid 2194 mappings = {} 2195 for i in range( 1, 29 ): # hosts 1 through 28 2196 # set up correct variables: 2197 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2) 2198 if i == 1: 2199 deviceId = "1000".zfill(16) 2200 elif i == 2: 2201 deviceId = "2000".zfill(16) 2202 elif i == 3: 2203 deviceId = "3000".zfill(16) 2204 elif i == 4: 2205 deviceId = "3004".zfill(16) 2206 elif i == 5: 2207 deviceId = "5000".zfill(16) 2208 elif i == 6: 2209 deviceId = "6000".zfill(16) 2210 elif i == 7: 2211 deviceId = "6007".zfill(16) 2212 elif i >= 8 and i <= 17: 2213 dpid = '3' + str( i ).zfill( 3 ) 2214 deviceId = dpid.zfill(16) 2215 elif i >= 18 and i <= 27: 2216 dpid = '6' + str( i ).zfill( 3 ) 2217 deviceId = dpid.zfill(16) 2218 elif i == 28: 2219 deviceId = "2800".zfill(16) 2220 mappings[ macId ] = deviceId 2221 if hosts[ controller ] or "Error" not in hosts[ controller ]: 2222 if hosts[ controller ] == []: 2223 main.log.warn( "There are no hosts discovered" ) 2224 zeroHosts = True 2225 else: 2226 for host in hosts[ controller ]: 2227 mac = None 2228 location = None 2229 device = None 2230 port = None 2231 try: 2232 mac = host.get( 'mac' ) 2233 assert mac, "mac field could not be found for this host object" 2234 2235 location = host.get( 'location' ) 2236 assert location, "location field could not be found for this host object" 2237 2238 # Trim the protocol identifier off deviceId 2239 device = str( location.get( 'elementId' ) ).split(':')[1] 2240 assert device, "elementId field could not be found for this host location object" 2241 2242 port = location.get( 'port' ) 2243 assert port, "port field could not be found for this host location object" 2244 2245 # Now check if this matches where they should be 2246 if mac and device and port: 2247 if str( port ) != "1": 2248 main.log.error( "The attachment port is incorrect for " + 2249 "host " + str( mac ) + 2250 ". Expected: 1 Actual: " + str( port) ) 2251 hostAttachment = False 2252 if device != mappings[ str( mac ) ]: 2253 main.log.error( "The attachment device is incorrect for " + 2254 "host " + str( mac ) + 2255 ". Expected: " + mappings[ str( mac ) ] + 2256 " Actual: " + device ) 2257 hostAttachment = False 2258 else: 2259 hostAttachment = False 2260 except AssertionError: 2261 main.log.exception( "Json object not as expected" ) 2262 main.log.error( repr( host ) ) 2263 hostAttachment = False 2264 else: 2265 main.log.error( "No hosts json output or \"Error\"" + 2266 " in output. hosts = " + 2267 repr( hosts[ controller ] ) ) 2268 if zeroHosts is False: 2269 hostAttachment = True 2270 2271 # END CHECKING HOST ATTACHMENT POINTS 2272 devicesResults = devicesResults and currentDevicesResult 2273 linksResults = linksResults and currentLinksResult 2274 hostsResults = hostsResults and currentHostsResult 2275 hostAttachmentResults = hostAttachmentResults and\ 2276 hostAttachment 2277 topoResult = ( devicesResults and linksResults 2278 and hostsResults and ipResult and 2279 hostAttachmentResults ) 2280 2281 # Compare json objects for hosts and dataplane clusters 2282 2283 # hosts 2284 main.step( "Hosts view is consistent across all ONOS nodes" ) 2285 consistentHostsResult = main.TRUE 2286 for controller in range( len( hosts ) ): 2287 controllerStr = str( controller + 1 ) 2288 if "Error" not in hosts[ controller ]: 2289 if hosts[ controller ] == hosts[ 0 ]: 2290 continue 2291 else: # hosts not consistent 2292 main.log.error( "hosts from ONOS" + controllerStr + 2293 " is inconsistent with ONOS1" ) 2294 main.log.warn( repr( hosts[ controller ] ) ) 2295 consistentHostsResult = main.FALSE 2296 2297 else: 2298 main.log.error( "Error in getting ONOS hosts from ONOS" + 2299 controllerStr ) 2300 consistentHostsResult = main.FALSE 2301 main.log.warn( "ONOS" + controllerStr + 2302 " hosts response: " + 2303 repr( hosts[ controller ] ) ) 2304 utilities.assert_equals( 2305 expect=main.TRUE, 2306 actual=consistentHostsResult, 2307 onpass="Hosts view is consistent across all ONOS nodes", 2308 onfail="ONOS nodes have different views of hosts" ) 2309 2310 main.step( "Hosts information is correct" ) 2311 hostsResults = hostsResults and ipResult 2312 utilities.assert_equals( 2313 expect=main.TRUE, 2314 actual=hostsResults, 2315 onpass="Host information is correct", 2316 onfail="Host information is incorrect" ) 2317 2318 main.step( "Host attachment points to the network" ) 2319 utilities.assert_equals( 2320 expect=True, 2321 actual=hostAttachmentResults, 2322 onpass="Hosts are correctly attached to the network", 2323 onfail="ONOS did not correctly attach hosts to the network" ) 2324 2325 # Strongly connected clusters of devices 2326 main.step( "Clusters view is consistent across all ONOS nodes" ) 2327 consistentClustersResult = main.TRUE 2328 for controller in range( len( clusters ) ): 2329 controllerStr = str( controller + 1 ) 2330 if "Error" not in clusters[ controller ]: 2331 if clusters[ controller ] == clusters[ 0 ]: 2332 continue 2333 else: # clusters not consistent 2334 main.log.error( "clusters from ONOS" + 2335 controllerStr + 2336 " is inconsistent with ONOS1" ) 2337 consistentClustersResult = main.FALSE 2338 2339 else: 2340 main.log.error( "Error in getting dataplane clusters " + 2341 "from ONOS" + controllerStr ) 2342 consistentClustersResult = main.FALSE 2343 main.log.warn( "ONOS" + controllerStr + 2344 " clusters response: " + 2345 repr( clusters[ controller ] ) ) 2346 utilities.assert_equals( 2347 expect=main.TRUE, 2348 actual=consistentClustersResult, 2349 onpass="Clusters view is consistent across all ONOS nodes", 2350 onfail="ONOS nodes have different views of clusters" ) 2351 2352 main.step( "There is only one SCC" ) 2353 # there should always only be one cluster 2354 try: 2355 numClusters = len( json.loads( clusters[ 0 ] ) ) 2356 except ( ValueError, TypeError ): 2357 main.log.exception( "Error parsing clusters[0]: " + 2358 repr( clusters[0] ) ) 2359 clusterResults = main.FALSE 2360 if numClusters == 1: 2361 clusterResults = main.TRUE 2362 utilities.assert_equals( 2363 expect=1, 2364 actual=numClusters, 2365 onpass="ONOS shows 1 SCC", 2366 onfail="ONOS shows " + str( numClusters ) + " SCCs" ) 2367 2368 topoResult = ( devicesResults and linksResults 2369 and hostsResults and consistentHostsResult 2370 and consistentClustersResult and clusterResults 2371 and ipResult and hostAttachmentResults ) 2372 2373 topoResult = topoResult and int( count <= 2 ) 2374 note = "note it takes about " + str( int( cliTime ) ) + \ 2375 " seconds for the test to make all the cli calls to fetch " +\ 2376 "the topology from each ONOS instance" 2377 main.log.info( 2378 "Very crass estimate for topology discovery/convergence( " + 2379 str( note ) + " ): " + str( elapsed ) + " seconds, " + 2380 str( count ) + " tries" ) 2381 2382 main.step( "Device information is correct" ) 2383 utilities.assert_equals( 2384 expect=main.TRUE, 2385 actual=devicesResults, 2386 onpass="Device information is correct", 2387 onfail="Device information is incorrect" ) 2388 2389 main.step( "Links are correct" ) 2390 utilities.assert_equals( 2391 expect=main.TRUE, 2392 actual=linksResults, 2393 onpass="Link are correct", 2394 onfail="Links are incorrect" ) 2395 2396 main.step( "Hosts are correct" ) 2397 utilities.assert_equals( 2398 expect=main.TRUE, 2399 actual=hostsResults, 2400 onpass="Hosts are correct", 2401 onfail="Hosts are incorrect" ) 2402 2403 # FIXME: move this to an ONOS state case 2404 main.step( "Checking ONOS nodes" ) 2405 nodesOutput = [] 2406 nodeResults = main.TRUE 2407 threads = [] 2408 for i in range( main.numCtrls ): 2409 t = main.Thread( target=main.CLIs[i].nodes, 2410 name="nodes-" + str( i ), 2411 args=[ ] ) 2412 threads.append( t ) 2413 t.start() 2414 2415 for t in threads: 2416 t.join() 2417 nodesOutput.append( t.result ) 2418 ips = [ node.ip_address for node in main.nodes ] 2419 for i in nodesOutput: 2420 try: 2421 current = json.loads( i ) 2422 for node in current: 2423 currentResult = main.FALSE 2424 if node['ip'] in ips: # node in nodes() output is in cell 2425 if node['state'] == 'ACTIVE': 2426 currentResult = main.TRUE 2427 else: 2428 main.log.error( "Error in ONOS node availability" ) 2429 main.log.error( 2430 json.dumps( current, 2431 sort_keys=True, 2432 indent=4, 2433 separators=( ',', ': ' ) ) ) 2434 break 2435 nodeResults = nodeResults and currentResult 2436 except ( ValueError, TypeError ): 2437 main.log.error( "Error parsing nodes output" ) 2438 main.log.warn( repr( i ) ) 2439 utilities.assert_equals( expect=main.TRUE, actual=nodeResults, 2440 onpass="Nodes check successful", 2441 onfail="Nodes check NOT successful" )
2442
2443 - def CASE9( self, main ):
2444 """ 2445 Link s3-s28 down 2446 """ 2447 import time 2448 assert main.numCtrls, "main.numCtrls not defined" 2449 assert main, "main not defined" 2450 assert utilities.assert_equals, "utilities.assert_equals not defined" 2451 assert main.CLIs, "main.CLIs not defined" 2452 assert main.nodes, "main.nodes not defined" 2453 # NOTE: You should probably run a topology check after this 2454 2455 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] ) 2456 2457 description = "Turn off a link to ensure that Link Discovery " +\ 2458 "is working properly" 2459 main.case( description ) 2460 2461 main.step( "Kill Link between s3 and s28" ) 2462 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" ) 2463 main.log.info( "Waiting " + str( linkSleep ) + 2464 " seconds for link down to be discovered" ) 2465 time.sleep( linkSleep ) 2466 utilities.assert_equals( expect=main.TRUE, actual=LinkDown, 2467 onpass="Link down successful", 2468 onfail="Failed to bring link down" )
2469 # TODO do some sort of check here 2470
2471 - def CASE10( self, main ):
2472 """ 2473 Link s3-s28 up 2474 """ 2475 import time 2476 assert main.numCtrls, "main.numCtrls not defined" 2477 assert main, "main not defined" 2478 assert utilities.assert_equals, "utilities.assert_equals not defined" 2479 assert main.CLIs, "main.CLIs not defined" 2480 assert main.nodes, "main.nodes not defined" 2481 # NOTE: You should probably run a topology check after this 2482 2483 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] ) 2484 2485 description = "Restore a link to ensure that Link Discovery is " + \ 2486 "working properly" 2487 main.case( description ) 2488 2489 main.step( "Bring link between s3 and s28 back up" ) 2490 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" ) 2491 main.log.info( "Waiting " + str( linkSleep ) + 2492 " seconds for link up to be discovered" ) 2493 time.sleep( linkSleep ) 2494 utilities.assert_equals( expect=main.TRUE, actual=LinkUp, 2495 onpass="Link up successful", 2496 onfail="Failed to bring link up" )
2497 # TODO do some sort of check here 2498
2499 - def CASE11( self, main ):
2500 """ 2501 Switch Down 2502 """ 2503 # NOTE: You should probably run a topology check after this 2504 import time 2505 assert main.numCtrls, "main.numCtrls not defined" 2506 assert main, "main not defined" 2507 assert utilities.assert_equals, "utilities.assert_equals not defined" 2508 assert main.CLIs, "main.CLIs not defined" 2509 assert main.nodes, "main.nodes not defined" 2510 2511 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) 2512 2513 description = "Killing a switch to ensure it is discovered correctly" 2514 main.case( description ) 2515 switch = main.params[ 'kill' ][ 'switch' ] 2516 switchDPID = main.params[ 'kill' ][ 'dpid' ] 2517 2518 # TODO: Make this switch parameterizable 2519 main.step( "Kill " + switch ) 2520 main.log.info( "Deleting " + switch ) 2521 main.Mininet1.delSwitch( switch ) 2522 main.log.info( "Waiting " + str( switchSleep ) + 2523 " seconds for switch down to be discovered" ) 2524 time.sleep( switchSleep ) 2525 device = main.ONOScli1.getDevice( dpid=switchDPID ) 2526 # Peek at the deleted switch 2527 main.log.warn( str( device ) ) 2528 result = main.FALSE 2529 if device and device[ 'available' ] is False: 2530 result = main.TRUE 2531 utilities.assert_equals( expect=main.TRUE, actual=result, 2532 onpass="Kill switch successful", 2533 onfail="Failed to kill switch?" )
2534
2535 - def CASE12( self, main ):
2536 """ 2537 Switch Up 2538 """ 2539 # NOTE: You should probably run a topology check after this 2540 import time 2541 assert main.numCtrls, "main.numCtrls not defined" 2542 assert main, "main not defined" 2543 assert utilities.assert_equals, "utilities.assert_equals not defined" 2544 assert main.CLIs, "main.CLIs not defined" 2545 assert main.nodes, "main.nodes not defined" 2546 assert ONOS1Port, "ONOS1Port not defined" 2547 assert ONOS2Port, "ONOS2Port not defined" 2548 assert ONOS3Port, "ONOS3Port not defined" 2549 assert ONOS4Port, "ONOS4Port not defined" 2550 assert ONOS5Port, "ONOS5Port not defined" 2551 assert ONOS6Port, "ONOS6Port not defined" 2552 assert ONOS7Port, "ONOS7Port not defined" 2553 2554 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) 2555 switch = main.params[ 'kill' ][ 'switch' ] 2556 switchDPID = main.params[ 'kill' ][ 'dpid' ] 2557 links = main.params[ 'kill' ][ 'links' ].split() 2558 description = "Adding a switch to ensure it is discovered correctly" 2559 main.case( description ) 2560 2561 main.step( "Add back " + switch ) 2562 main.Mininet1.addSwitch( switch, dpid=switchDPID ) 2563 for peer in links: 2564 main.Mininet1.addLink( switch, peer ) 2565 ipList = [] 2566 for i in range( main.numCtrls ): 2567 ipList.append( main.nodes[ i ].ip_address ) 2568 main.Mininet1.assignSwController( sw=switch, ip=ipList ) 2569 main.log.info( "Waiting " + str( switchSleep ) + 2570 " seconds for switch up to be discovered" ) 2571 time.sleep( switchSleep ) 2572 device = main.ONOScli1.getDevice( dpid=switchDPID ) 2573 # Peek at the deleted switch 2574 main.log.warn( str( device ) ) 2575 result = main.FALSE 2576 if device and device[ 'available' ]: 2577 result = main.TRUE 2578 utilities.assert_equals( expect=main.TRUE, actual=result, 2579 onpass="add switch successful", 2580 onfail="Failed to add switch?" )
2581
2582 - def CASE13( self, main ):
2583 """ 2584 Clean up 2585 """ 2586 import os 2587 import time 2588 assert main.numCtrls, "main.numCtrls not defined" 2589 assert main, "main not defined" 2590 assert utilities.assert_equals, "utilities.assert_equals not defined" 2591 assert main.CLIs, "main.CLIs not defined" 2592 assert main.nodes, "main.nodes not defined" 2593 2594 # printing colors to terminal 2595 colors = { 'cyan': '\033[96m', 'purple': '\033[95m', 2596 'blue': '\033[94m', 'green': '\033[92m', 2597 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' } 2598 main.case( "Test Cleanup" ) 2599 main.step( "Killing tcpdumps" ) 2600 main.Mininet2.stopTcpdump() 2601 2602 testname = main.TEST 2603 if main.params[ 'BACKUP' ] == "True": 2604 main.step( "Copying MN pcap and ONOS log files to test station" ) 2605 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ] 2606 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ] 2607 # NOTE: MN Pcap file is being saved to ~/packet_captures 2608 # scp this file as MN and TestON aren't necessarily the same vm 2609 # FIXME: scp 2610 # mn files 2611 # TODO: Load these from params 2612 # NOTE: must end in / 2613 logFolder = "/opt/onos/log/" 2614 logFiles = [ "karaf.log", "karaf.log.1" ] 2615 # NOTE: must end in / 2616 dstDir = "~/packet_captures/" 2617 for f in logFiles: 2618 for node in main.nodes: 2619 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address + 2620 ":" + logFolder + f + " " + 2621 teststationUser + "@" + 2622 teststationIP + ":" + 2623 dstDir + str( testname ) + 2624 "-" + node.name + "-" + f ) 2625 main.ONOSbench.handle.expect( "\$" ) 2626 2627 # std*.log's 2628 # NOTE: must end in / 2629 logFolder = "/opt/onos/var/" 2630 logFiles = [ "stderr.log", "stdout.log" ] 2631 # NOTE: must end in / 2632 dstDir = "~/packet_captures/" 2633 for f in logFiles: 2634 for node in main.nodes: 2635 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address + 2636 ":" + logFolder + f + " " + 2637 teststationUser + "@" + 2638 teststationIP + ":" + 2639 dstDir + str( testname ) + 2640 "-" + node.name + "-" + f ) 2641 main.ONOSbench.handle.expect( "\$" ) 2642 # sleep so scp can finish 2643 time.sleep( 10 ) 2644 main.step( "Packing and rotating pcap archives" ) 2645 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) ) 2646 2647 main.step( "Stopping Mininet" ) 2648 mnResult = main.Mininet1.stopNet() 2649 utilities.assert_equals( expect=main.TRUE, actual=mnResult, 2650 onpass="Mininet stopped", 2651 onfail="MN cleanup NOT successful" ) 2652 2653 main.step( "Checking ONOS Logs for errors" ) 2654 for node in main.nodes: 2655 print colors[ 'purple' ] + "Checking logs for errors on " + \ 2656 node.name + ":" + colors[ 'end' ] 2657 print main.ONOSbench.checkLogs( node.ip_address ) 2658 2659 try: 2660 timerLog = open( main.logdir + "/Timers.csv", 'w') 2661 # Overwrite with empty line and close 2662 labels = "Gossip Intents" 2663 data = str( gossipTime ) 2664 timerLog.write( labels + "\n" + data ) 2665 timerLog.close() 2666 except NameError, e: 2667 main.log.exception(e)
2668
2669 - def CASE14( self, main ):
2670 """ 2671 start election app on all onos nodes 2672 """ 2673 assert main.numCtrls, "main.numCtrls not defined" 2674 assert main, "main not defined" 2675 assert utilities.assert_equals, "utilities.assert_equals not defined" 2676 assert main.CLIs, "main.CLIs not defined" 2677 assert main.nodes, "main.nodes not defined" 2678 2679 main.case("Start Leadership Election app") 2680 main.step( "Install leadership election app" ) 2681 appResult = main.ONOScli1.activateApp( "org.onosproject.election" ) 2682 utilities.assert_equals( 2683 expect=main.TRUE, 2684 actual=appResult, 2685 onpass="Election app installed", 2686 onfail="Something went wrong with installing Leadership election" ) 2687 2688 main.step( "Run for election on each node" ) 2689 leaderResult = main.TRUE 2690 leaders = [] 2691 for cli in main.CLIs: 2692 cli.electionTestRun() 2693 for cli in main.CLIs: 2694 leader = cli.electionTestLeader() 2695 if leader is None or leader == main.FALSE: 2696 main.log.error( cli.name + ": Leader for the election app " + 2697 "should be an ONOS node, instead got '" + 2698 str( leader ) + "'" ) 2699 leaderResult = main.FALSE 2700 leaders.append( leader ) 2701 utilities.assert_equals( 2702 expect=main.TRUE, 2703 actual=leaderResult, 2704 onpass="Successfully ran for leadership", 2705 onfail="Failed to run for leadership" ) 2706 2707 main.step( "Check that each node shows the same leader" ) 2708 sameLeader = main.TRUE 2709 if len( set( leaders ) ) != 1: 2710 sameLeader = main.FALSE 2711 main.log.error( "Results of electionTestLeader is order of main.CLIs:" + 2712 str( leaders ) ) 2713 utilities.assert_equals( 2714 expect=main.TRUE, 2715 actual=sameLeader, 2716 onpass="Leadership is consistent for the election topic", 2717 onfail="Nodes have different leaders" )
2718
2719 - def CASE15( self, main ):
2720 """ 2721 Check that Leadership Election is still functional 2722 """ 2723 import time 2724 assert main.numCtrls, "main.numCtrls not defined" 2725 assert main, "main not defined" 2726 assert utilities.assert_equals, "utilities.assert_equals not defined" 2727 assert main.CLIs, "main.CLIs not defined" 2728 assert main.nodes, "main.nodes not defined" 2729 2730 leaderResult = main.TRUE 2731 description = "Check that Leadership Election is still functional" 2732 main.case( description ) 2733 2734 main.step( "Check that each node shows the same leader" ) 2735 sameLeader = main.TRUE 2736 leaders = [] 2737 for cli in main.CLIs: 2738 leader = cli.electionTestLeader() 2739 leaders.append( leader ) 2740 if len( set( leaders ) ) != 1: 2741 sameLeader = main.FALSE 2742 main.log.error( "Results of electionTestLeader is order of main.CLIs:" + 2743 str( leaders ) ) 2744 utilities.assert_equals( 2745 expect=main.TRUE, 2746 actual=sameLeader, 2747 onpass="Leadership is consistent for the election topic", 2748 onfail="Nodes have different leaders" ) 2749 2750 main.step( "Find current leader and withdraw" ) 2751 leader = main.ONOScli1.electionTestLeader() 2752 # do some sanity checking on leader before using it 2753 withdrawResult = main.FALSE 2754 if leader is None or leader == main.FALSE: 2755 main.log.error( 2756 "Leader for the election app should be an ONOS node," + 2757 "instead got '" + str( leader ) + "'" ) 2758 leaderResult = main.FALSE 2759 oldLeader = None 2760 for i in range( len( main.CLIs ) ): 2761 if leader == main.nodes[ i ].ip_address: 2762 oldLeader = main.CLIs[ i ] 2763 break 2764 else: # FOR/ELSE statement 2765 main.log.error( "Leader election, could not find current leader" ) 2766 if oldLeader: 2767 withdrawResult = oldLeader.electionTestWithdraw() 2768 utilities.assert_equals( 2769 expect=main.TRUE, 2770 actual=withdrawResult, 2771 onpass="Node was withdrawn from election", 2772 onfail="Node was not withdrawn from election" ) 2773 2774 main.step( "Make sure new leader is elected" ) 2775 # FIXME: use threads 2776 leaderList = [] 2777 for cli in main.CLIs: 2778 leaderN = cli.electionTestLeader() 2779 leaderList.append( leaderN ) 2780 if leaderN == leader: 2781 main.log.error( cli.name + " still sees " + str( leader ) + 2782 " as leader after they withdrew" ) 2783 leaderResult = main.FALSE 2784 elif leaderN == main.FALSE: 2785 # error in response 2786 # TODO: add check for "Command not found:" in the driver, this 2787 # means the app isn't loaded 2788 main.log.error( "Something is wrong with " + 2789 "electionTestLeader function, " + 2790 "check the error logs" ) 2791 leaderResult = main.FALSE 2792 elif leaderN is None: 2793 # node may not have recieved the event yet 2794 time.sleep(7) 2795 leaderN = cli.electionTestLeader() 2796 leaderList.pop() 2797 leaderList.append( leaderN ) 2798 consistentLeader = main.FALSE 2799 if len( set( leaderList ) ) == 1: 2800 main.log.info( "Each Election-app sees '" + 2801 str( leaderList[ 0 ] ) + 2802 "' as the leader" ) 2803 consistentLeader = main.TRUE 2804 else: 2805 main.log.error( 2806 "Inconsistent responses for leader of Election-app:" ) 2807 for n in range( len( leaderList ) ): 2808 main.log.error( "ONOS" + str( n + 1 ) + " response: " + 2809 str( leaderList[ n ] ) ) 2810 leaderResult = leaderResult and consistentLeader 2811 utilities.assert_equals( 2812 expect=main.TRUE, 2813 actual=leaderResult, 2814 onpass="Leadership election passed", 2815 onfail="Something went wrong with Leadership election" ) 2816 2817 main.step( "Run for election on old leader( just so everyone " + 2818 "is in the hat )" ) 2819 if oldLeader: 2820 runResult = oldLeader.electionTestRun() 2821 else: 2822 runResult = main.FALSE 2823 utilities.assert_equals( 2824 expect=main.TRUE, 2825 actual=runResult, 2826 onpass="App re-ran for election", 2827 onfail="App failed to run for election" ) 2828 2829 main.step( "Leader did not change when old leader re-ran" ) 2830 afterRun = main.ONOScli1.electionTestLeader() 2831 # verify leader didn't just change 2832 if afterRun == leaderList[ 0 ]: 2833 afterResult = main.TRUE 2834 else: 2835 afterResult = main.FALSE 2836 2837 utilities.assert_equals( 2838 expect=main.TRUE, 2839 actual=afterResult, 2840 onpass="Old leader successfully re-ran for election", 2841 onfail="Something went wrong with Leadership election after " + 2842 "the old leader re-ran for election" )
2843
2844 - def CASE16( self, main ):
2845 """ 2846 Install Distributed Primitives app 2847 """ 2848 import time 2849 assert main.numCtrls, "main.numCtrls not defined" 2850 assert main, "main not defined" 2851 assert utilities.assert_equals, "utilities.assert_equals not defined" 2852 assert main.CLIs, "main.CLIs not defined" 2853 assert main.nodes, "main.nodes not defined" 2854 2855 # Variables for the distributed primitives tests 2856 global pCounterName 2857 global iCounterName 2858 global pCounterValue 2859 global iCounterValue 2860 global onosSet 2861 global onosSetName 2862 pCounterName = "TestON-Partitions" 2863 iCounterName = "TestON-inMemory" 2864 pCounterValue = 0 2865 iCounterValue = 0 2866 onosSet = set([]) 2867 onosSetName = "TestON-set" 2868 2869 description = "Install Primitives app" 2870 main.case( description ) 2871 main.step( "Install Primitives app" ) 2872 appName = "org.onosproject.distributedprimitives" 2873 appResults = main.CLIs[0].activateApp( appName ) 2874 utilities.assert_equals( expect=main.TRUE, 2875 actual=appResults, 2876 onpass="Primitives app activated", 2877 onfail="Primitives app not activated" ) 2878 time.sleep( 5 ) # To allow all nodes to activate
2879
2880 - def CASE17( self, main ):
2881 """ 2882 Check for basic functionality with distributed primitives 2883 """ 2884 # Make sure variables are defined/set 2885 assert main.numCtrls, "main.numCtrls not defined" 2886 assert main, "main not defined" 2887 assert utilities.assert_equals, "utilities.assert_equals not defined" 2888 assert main.CLIs, "main.CLIs not defined" 2889 assert main.nodes, "main.nodes not defined" 2890 assert pCounterName, "pCounterName not defined" 2891 assert iCounterName, "iCounterName not defined" 2892 assert onosSetName, "onosSetName not defined" 2893 # NOTE: assert fails if value is 0/None/Empty/False 2894 try: 2895 pCounterValue 2896 except NameError: 2897 main.log.error( "pCounterValue not defined, setting to 0" ) 2898 pCounterValue = 0 2899 try: 2900 iCounterValue 2901 except NameError: 2902 main.log.error( "iCounterValue not defined, setting to 0" ) 2903 iCounterValue = 0 2904 try: 2905 onosSet 2906 except NameError: 2907 main.log.error( "onosSet not defined, setting to empty Set" ) 2908 onosSet = set([]) 2909 # Variables for the distributed primitives tests. These are local only 2910 addValue = "a" 2911 addAllValue = "a b c d e f" 2912 retainValue = "c d e f" 2913 2914 description = "Check for basic functionality with distributed " +\ 2915 "primitives" 2916 main.case( description ) 2917 main.caseExplanation = "Test the methods of the distributed " +\ 2918 "primitives (counters and sets) throught the cli" 2919 # DISTRIBUTED ATOMIC COUNTERS 2920 # Partitioned counters 2921 main.step( "Increment then get a default counter on each node" ) 2922 pCounters = [] 2923 threads = [] 2924 addedPValues = [] 2925 for i in range( main.numCtrls ): 2926 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 2927 name="counterAddAndGet-" + str( i ), 2928 args=[ pCounterName ] ) 2929 pCounterValue += 1 2930 addedPValues.append( pCounterValue ) 2931 threads.append( t ) 2932 t.start() 2933 2934 for t in threads: 2935 t.join() 2936 pCounters.append( t.result ) 2937 # Check that counter incremented numController times 2938 pCounterResults = True 2939 for i in addedPValues: 2940 tmpResult = i in pCounters 2941 pCounterResults = pCounterResults and tmpResult 2942 if not tmpResult: 2943 main.log.error( str( i ) + " is not in partitioned " 2944 "counter incremented results" ) 2945 utilities.assert_equals( expect=True, 2946 actual=pCounterResults, 2947 onpass="Default counter incremented", 2948 onfail="Error incrementing default" + 2949 " counter" ) 2950 2951 main.step( "Get then Increment a default counter on each node" ) 2952 pCounters = [] 2953 threads = [] 2954 addedPValues = [] 2955 for i in range( main.numCtrls ): 2956 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 2957 name="counterGetAndAdd-" + str( i ), 2958 args=[ pCounterName ] ) 2959 addedPValues.append( pCounterValue ) 2960 pCounterValue += 1 2961 threads.append( t ) 2962 t.start() 2963 2964 for t in threads: 2965 t.join() 2966 pCounters.append( t.result ) 2967 # Check that counter incremented numController times 2968 pCounterResults = True 2969 for i in addedPValues: 2970 tmpResult = i in pCounters 2971 pCounterResults = pCounterResults and tmpResult 2972 if not tmpResult: 2973 main.log.error( str( i ) + " is not in partitioned " 2974 "counter incremented results" ) 2975 utilities.assert_equals( expect=True, 2976 actual=pCounterResults, 2977 onpass="Default counter incremented", 2978 onfail="Error incrementing default" + 2979 " counter" ) 2980 2981 main.step( "Counters we added have the correct values" ) 2982 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue ) 2983 utilities.assert_equals( expect=main.TRUE, 2984 actual=incrementCheck, 2985 onpass="Added counters are correct", 2986 onfail="Added counters are incorrect" ) 2987 2988 main.step( "Add -8 to then get a default counter on each node" ) 2989 pCounters = [] 2990 threads = [] 2991 addedPValues = [] 2992 for i in range( main.numCtrls ): 2993 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 2994 name="counterIncrement-" + str( i ), 2995 args=[ pCounterName ], 2996 kwargs={ "delta": -8 } ) 2997 pCounterValue += -8 2998 addedPValues.append( pCounterValue ) 2999 threads.append( t ) 3000 t.start() 3001 3002 for t in threads: 3003 t.join() 3004 pCounters.append( t.result ) 3005 # Check that counter incremented numController times 3006 pCounterResults = True 3007 for i in addedPValues: 3008 tmpResult = i in pCounters 3009 pCounterResults = pCounterResults and tmpResult 3010 if not tmpResult: 3011 main.log.error( str( i ) + " is not in partitioned " 3012 "counter incremented results" ) 3013 utilities.assert_equals( expect=True, 3014 actual=pCounterResults, 3015 onpass="Default counter incremented", 3016 onfail="Error incrementing default" + 3017 " counter" ) 3018 3019 main.step( "Add 5 to then get a default counter on each node" ) 3020 pCounters = [] 3021 threads = [] 3022 addedPValues = [] 3023 for i in range( main.numCtrls ): 3024 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3025 name="counterIncrement-" + str( i ), 3026 args=[ pCounterName ], 3027 kwargs={ "delta": 5 } ) 3028 pCounterValue += 5 3029 addedPValues.append( pCounterValue ) 3030 threads.append( t ) 3031 t.start() 3032 3033 for t in threads: 3034 t.join() 3035 pCounters.append( t.result ) 3036 # Check that counter incremented numController times 3037 pCounterResults = True 3038 for i in addedPValues: 3039 tmpResult = i in pCounters 3040 pCounterResults = pCounterResults and tmpResult 3041 if not tmpResult: 3042 main.log.error( str( i ) + " is not in partitioned " 3043 "counter incremented results" ) 3044 utilities.assert_equals( expect=True, 3045 actual=pCounterResults, 3046 onpass="Default counter incremented", 3047 onfail="Error incrementing default" + 3048 " counter" ) 3049 3050 main.step( "Get then add 5 to a default counter on each node" ) 3051 pCounters = [] 3052 threads = [] 3053 addedPValues = [] 3054 for i in range( main.numCtrls ): 3055 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3056 name="counterIncrement-" + str( i ), 3057 args=[ pCounterName ], 3058 kwargs={ "delta": 5 } ) 3059 addedPValues.append( pCounterValue ) 3060 pCounterValue += 5 3061 threads.append( t ) 3062 t.start() 3063 3064 for t in threads: 3065 t.join() 3066 pCounters.append( t.result ) 3067 # Check that counter incremented numController times 3068 pCounterResults = True 3069 for i in addedPValues: 3070 tmpResult = i in pCounters 3071 pCounterResults = pCounterResults and tmpResult 3072 if not tmpResult: 3073 main.log.error( str( i ) + " is not in partitioned " 3074 "counter incremented results" ) 3075 utilities.assert_equals( expect=True, 3076 actual=pCounterResults, 3077 onpass="Default counter incremented", 3078 onfail="Error incrementing default" + 3079 " counter" ) 3080 3081 main.step( "Counters we added have the correct values" ) 3082 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue ) 3083 utilities.assert_equals( expect=main.TRUE, 3084 actual=incrementCheck, 3085 onpass="Added counters are correct", 3086 onfail="Added counters are incorrect" ) 3087 3088 # In-Memory counters 3089 main.step( "Increment and get an in-memory counter on each node" ) 3090 iCounters = [] 3091 addedIValues = [] 3092 threads = [] 3093 for i in range( main.numCtrls ): 3094 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3095 name="icounterIncrement-" + str( i ), 3096 args=[ iCounterName ], 3097 kwargs={ "inMemory": True } ) 3098 iCounterValue += 1 3099 addedIValues.append( iCounterValue ) 3100 threads.append( t ) 3101 t.start() 3102 3103 for t in threads: 3104 t.join() 3105 iCounters.append( t.result ) 3106 # Check that counter incremented numController times 3107 iCounterResults = True 3108 for i in addedIValues: 3109 tmpResult = i in iCounters 3110 iCounterResults = iCounterResults and tmpResult 3111 if not tmpResult: 3112 main.log.error( str( i ) + " is not in the in-memory " 3113 "counter incremented results" ) 3114 utilities.assert_equals( expect=True, 3115 actual=iCounterResults, 3116 onpass="In-memory counter incremented", 3117 onfail="Error incrementing in-memory" + 3118 " counter" ) 3119 3120 main.step( "Get then Increment a in-memory counter on each node" ) 3121 iCounters = [] 3122 threads = [] 3123 addedIValues = [] 3124 for i in range( main.numCtrls ): 3125 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3126 name="counterGetAndAdd-" + str( i ), 3127 args=[ iCounterName ], 3128 kwargs={ "inMemory": True } ) 3129 addedIValues.append( iCounterValue ) 3130 iCounterValue += 1 3131 threads.append( t ) 3132 t.start() 3133 3134 for t in threads: 3135 t.join() 3136 iCounters.append( t.result ) 3137 # Check that counter incremented numController times 3138 iCounterResults = True 3139 for i in addedIValues: 3140 tmpResult = i in iCounters 3141 iCounterResults = iCounterResults and tmpResult 3142 if not tmpResult: 3143 main.log.error( str( i ) + " is not in in-memory " 3144 "counter incremented results" ) 3145 utilities.assert_equals( expect=True, 3146 actual=iCounterResults, 3147 onpass="In-memory counter incremented", 3148 onfail="Error incrementing in-memory" + 3149 " counter" ) 3150 3151 main.step( "Counters we added have the correct values" ) 3152 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 3153 utilities.assert_equals( expect=main.TRUE, 3154 actual=incrementCheck, 3155 onpass="Added counters are correct", 3156 onfail="Added counters are incorrect" ) 3157 3158 main.step( "Add -8 to then get a in-memory counter on each node" ) 3159 iCounters = [] 3160 threads = [] 3161 addedIValues = [] 3162 for i in range( main.numCtrls ): 3163 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3164 name="counterIncrement-" + str( i ), 3165 args=[ iCounterName ], 3166 kwargs={ "delta": -8, "inMemory": True } ) 3167 iCounterValue += -8 3168 addedIValues.append( iCounterValue ) 3169 threads.append( t ) 3170 t.start() 3171 3172 for t in threads: 3173 t.join() 3174 iCounters.append( t.result ) 3175 # Check that counter incremented numController times 3176 iCounterResults = True 3177 for i in addedIValues: 3178 tmpResult = i in iCounters 3179 iCounterResults = iCounterResults and tmpResult 3180 if not tmpResult: 3181 main.log.error( str( i ) + " is not in in-memory " 3182 "counter incremented results" ) 3183 utilities.assert_equals( expect=True, 3184 actual=pCounterResults, 3185 onpass="In-memory counter incremented", 3186 onfail="Error incrementing in-memory" + 3187 " counter" ) 3188 3189 main.step( "Add 5 to then get a in-memory counter on each node" ) 3190 iCounters = [] 3191 threads = [] 3192 addedIValues = [] 3193 for i in range( main.numCtrls ): 3194 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3195 name="counterIncrement-" + str( i ), 3196 args=[ iCounterName ], 3197 kwargs={ "delta": 5, "inMemory": True } ) 3198 iCounterValue += 5 3199 addedIValues.append( iCounterValue ) 3200 threads.append( t ) 3201 t.start() 3202 3203 for t in threads: 3204 t.join() 3205 iCounters.append( t.result ) 3206 # Check that counter incremented numController times 3207 iCounterResults = True 3208 for i in addedIValues: 3209 tmpResult = i in iCounters 3210 iCounterResults = iCounterResults and tmpResult 3211 if not tmpResult: 3212 main.log.error( str( i ) + " is not in in-memory " 3213 "counter incremented results" ) 3214 utilities.assert_equals( expect=True, 3215 actual=pCounterResults, 3216 onpass="In-memory counter incremented", 3217 onfail="Error incrementing in-memory" + 3218 " counter" ) 3219 3220 main.step( "Get then add 5 to a in-memory counter on each node" ) 3221 iCounters = [] 3222 threads = [] 3223 addedIValues = [] 3224 for i in range( main.numCtrls ): 3225 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3226 name="counterIncrement-" + str( i ), 3227 args=[ iCounterName ], 3228 kwargs={ "delta": 5, "inMemory": True } ) 3229 addedIValues.append( iCounterValue ) 3230 iCounterValue += 5 3231 threads.append( t ) 3232 t.start() 3233 3234 for t in threads: 3235 t.join() 3236 iCounters.append( t.result ) 3237 # Check that counter incremented numController times 3238 iCounterResults = True 3239 for i in addedIValues: 3240 tmpResult = i in iCounters 3241 iCounterResults = iCounterResults and tmpResult 3242 if not tmpResult: 3243 main.log.error( str( i ) + " is not in in-memory " 3244 "counter incremented results" ) 3245 utilities.assert_equals( expect=True, 3246 actual=iCounterResults, 3247 onpass="In-memory counter incremented", 3248 onfail="Error incrementing in-memory" + 3249 " counter" ) 3250 3251 main.step( "Counters we added have the correct values" ) 3252 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 3253 utilities.assert_equals( expect=main.TRUE, 3254 actual=incrementCheck, 3255 onpass="Added counters are correct", 3256 onfail="Added counters are incorrect" ) 3257 3258 main.step( "Check counters are consistant across nodes" ) 3259 onosCounters = [] 3260 threads = [] 3261 for i in range( main.numCtrls ): 3262 t = main.Thread( target=main.CLIs[i].counters, 3263 name="counters-" + str( i ) ) 3264 threads.append( t ) 3265 t.start() 3266 for t in threads: 3267 t.join() 3268 onosCounters.append( t.result ) 3269 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ] 3270 if all( tmp ): 3271 main.log.info( "Counters are consistent across all nodes" ) 3272 consistentCounterResults = main.TRUE 3273 else: 3274 main.log.error( "Counters are not consistent across all nodes" ) 3275 consistentCounterResults = main.FALSE 3276 utilities.assert_equals( expect=main.TRUE, 3277 actual=consistentCounterResults, 3278 onpass="ONOS counters are consistent " + 3279 "across nodes", 3280 onfail="ONOS Counters are inconsistent " + 3281 "across nodes" ) 3282 3283 main.step( "Counters we added have the correct values" ) 3284 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 3285 incrementCheck = incrementCheck and \ 3286 main.Counters.counterCheck( iCounterName, iCounterValue ) 3287 utilities.assert_equals( expect=main.TRUE, 3288 actual=incrementCheck, 3289 onpass="Added counters are correct", 3290 onfail="Added counters are incorrect" ) 3291 3292 # DISTRIBUTED SETS 3293 main.step( "Distributed Set get" ) 3294 size = len( onosSet ) 3295 getResponses = [] 3296 threads = [] 3297 for i in range( main.numCtrls ): 3298 t = main.Thread( target=main.CLIs[i].setTestGet, 3299 name="setTestGet-" + str( i ), 3300 args=[ onosSetName ] ) 3301 threads.append( t ) 3302 t.start() 3303 for t in threads: 3304 t.join() 3305 getResponses.append( t.result ) 3306 3307 getResults = main.TRUE 3308 for i in range( main.numCtrls ): 3309 if isinstance( getResponses[ i ], list): 3310 current = set( getResponses[ i ] ) 3311 if len( current ) == len( getResponses[ i ] ): 3312 # no repeats 3313 if onosSet != current: 3314 main.log.error( "ONOS" + str( i + 1 ) + 3315 " has incorrect view" + 3316 " of set " + onosSetName + ":\n" + 3317 str( getResponses[ i ] ) ) 3318 main.log.debug( "Expected: " + str( onosSet ) ) 3319 main.log.debug( "Actual: " + str( current ) ) 3320 getResults = main.FALSE 3321 else: 3322 # error, set is not a set 3323 main.log.error( "ONOS" + str( i + 1 ) + 3324 " has repeat elements in" + 3325 " set " + onosSetName + ":\n" + 3326 str( getResponses[ i ] ) ) 3327 getResults = main.FALSE 3328 elif getResponses[ i ] == main.ERROR: 3329 getResults = main.FALSE 3330 utilities.assert_equals( expect=main.TRUE, 3331 actual=getResults, 3332 onpass="Set elements are correct", 3333 onfail="Set elements are incorrect" ) 3334 3335 main.step( "Distributed Set size" ) 3336 sizeResponses = [] 3337 threads = [] 3338 for i in range( main.numCtrls ): 3339 t = main.Thread( target=main.CLIs[i].setTestSize, 3340 name="setTestSize-" + str( i ), 3341 args=[ onosSetName ] ) 3342 threads.append( t ) 3343 t.start() 3344 for t in threads: 3345 t.join() 3346 sizeResponses.append( t.result ) 3347 3348 sizeResults = main.TRUE 3349 for i in range( main.numCtrls ): 3350 if size != sizeResponses[ i ]: 3351 sizeResults = main.FALSE 3352 main.log.error( "ONOS" + str( i + 1 ) + 3353 " expected a size of " + str( size ) + 3354 " for set " + onosSetName + 3355 " but got " + str( sizeResponses[ i ] ) ) 3356 utilities.assert_equals( expect=main.TRUE, 3357 actual=sizeResults, 3358 onpass="Set sizes are correct", 3359 onfail="Set sizes are incorrect" ) 3360 3361 main.step( "Distributed Set add()" ) 3362 onosSet.add( addValue ) 3363 addResponses = [] 3364 threads = [] 3365 for i in range( main.numCtrls ): 3366 t = main.Thread( target=main.CLIs[i].setTestAdd, 3367 name="setTestAdd-" + str( i ), 3368 args=[ onosSetName, addValue ] ) 3369 threads.append( t ) 3370 t.start() 3371 for t in threads: 3372 t.join() 3373 addResponses.append( t.result ) 3374 3375 # main.TRUE = successfully changed the set 3376 # main.FALSE = action resulted in no change in set 3377 # main.ERROR - Some error in executing the function 3378 addResults = main.TRUE 3379 for i in range( main.numCtrls ): 3380 if addResponses[ i ] == main.TRUE: 3381 # All is well 3382 pass 3383 elif addResponses[ i ] == main.FALSE: 3384 # Already in set, probably fine 3385 pass 3386 elif addResponses[ i ] == main.ERROR: 3387 # Error in execution 3388 addResults = main.FALSE 3389 else: 3390 # unexpected result 3391 addResults = main.FALSE 3392 if addResults != main.TRUE: 3393 main.log.error( "Error executing set add" ) 3394 3395 # Check if set is still correct 3396 size = len( onosSet ) 3397 getResponses = [] 3398 threads = [] 3399 for i in range( main.numCtrls ): 3400 t = main.Thread( target=main.CLIs[i].setTestGet, 3401 name="setTestGet-" + str( i ), 3402 args=[ onosSetName ] ) 3403 threads.append( t ) 3404 t.start() 3405 for t in threads: 3406 t.join() 3407 getResponses.append( t.result ) 3408 getResults = main.TRUE 3409 for i in range( main.numCtrls ): 3410 if isinstance( getResponses[ i ], list): 3411 current = set( getResponses[ i ] ) 3412 if len( current ) == len( getResponses[ i ] ): 3413 # no repeats 3414 if onosSet != current: 3415 main.log.error( "ONOS" + str( i + 1 ) + 3416 " has incorrect view" + 3417 " of set " + onosSetName + ":\n" + 3418 str( getResponses[ i ] ) ) 3419 main.log.debug( "Expected: " + str( onosSet ) ) 3420 main.log.debug( "Actual: " + str( current ) ) 3421 getResults = main.FALSE 3422 else: 3423 # error, set is not a set 3424 main.log.error( "ONOS" + str( i + 1 ) + 3425 " has repeat elements in" + 3426 " set " + onosSetName + ":\n" + 3427 str( getResponses[ i ] ) ) 3428 getResults = main.FALSE 3429 elif getResponses[ i ] == main.ERROR: 3430 getResults = main.FALSE 3431 sizeResponses = [] 3432 threads = [] 3433 for i in range( main.numCtrls ): 3434 t = main.Thread( target=main.CLIs[i].setTestSize, 3435 name="setTestSize-" + str( i ), 3436 args=[ onosSetName ] ) 3437 threads.append( t ) 3438 t.start() 3439 for t in threads: 3440 t.join() 3441 sizeResponses.append( t.result ) 3442 sizeResults = main.TRUE 3443 for i in range( main.numCtrls ): 3444 if size != sizeResponses[ i ]: 3445 sizeResults = main.FALSE 3446 main.log.error( "ONOS" + str( i + 1 ) + 3447 " expected a size of " + str( size ) + 3448 " for set " + onosSetName + 3449 " but got " + str( sizeResponses[ i ] ) ) 3450 addResults = addResults and getResults and sizeResults 3451 utilities.assert_equals( expect=main.TRUE, 3452 actual=addResults, 3453 onpass="Set add correct", 3454 onfail="Set add was incorrect" ) 3455 3456 main.step( "Distributed Set addAll()" ) 3457 onosSet.update( addAllValue.split() ) 3458 addResponses = [] 3459 threads = [] 3460 for i in range( main.numCtrls ): 3461 t = main.Thread( target=main.CLIs[i].setTestAdd, 3462 name="setTestAddAll-" + str( i ), 3463 args=[ onosSetName, addAllValue ] ) 3464 threads.append( t ) 3465 t.start() 3466 for t in threads: 3467 t.join() 3468 addResponses.append( t.result ) 3469 3470 # main.TRUE = successfully changed the set 3471 # main.FALSE = action resulted in no change in set 3472 # main.ERROR - Some error in executing the function 3473 addAllResults = main.TRUE 3474 for i in range( main.numCtrls ): 3475 if addResponses[ i ] == main.TRUE: 3476 # All is well 3477 pass 3478 elif addResponses[ i ] == main.FALSE: 3479 # Already in set, probably fine 3480 pass 3481 elif addResponses[ i ] == main.ERROR: 3482 # Error in execution 3483 addAllResults = main.FALSE 3484 else: 3485 # unexpected result 3486 addAllResults = main.FALSE 3487 if addAllResults != main.TRUE: 3488 main.log.error( "Error executing set addAll" ) 3489 3490 # Check if set is still correct 3491 size = len( onosSet ) 3492 getResponses = [] 3493 threads = [] 3494 for i in range( main.numCtrls ): 3495 t = main.Thread( target=main.CLIs[i].setTestGet, 3496 name="setTestGet-" + str( i ), 3497 args=[ onosSetName ] ) 3498 threads.append( t ) 3499 t.start() 3500 for t in threads: 3501 t.join() 3502 getResponses.append( t.result ) 3503 getResults = main.TRUE 3504 for i in range( main.numCtrls ): 3505 if isinstance( getResponses[ i ], list): 3506 current = set( getResponses[ i ] ) 3507 if len( current ) == len( getResponses[ i ] ): 3508 # no repeats 3509 if onosSet != current: 3510 main.log.error( "ONOS" + str( i + 1 ) + 3511 " has incorrect view" + 3512 " of set " + onosSetName + ":\n" + 3513 str( getResponses[ i ] ) ) 3514 main.log.debug( "Expected: " + str( onosSet ) ) 3515 main.log.debug( "Actual: " + str( current ) ) 3516 getResults = main.FALSE 3517 else: 3518 # error, set is not a set 3519 main.log.error( "ONOS" + str( i + 1 ) + 3520 " has repeat elements in" + 3521 " set " + onosSetName + ":\n" + 3522 str( getResponses[ i ] ) ) 3523 getResults = main.FALSE 3524 elif getResponses[ i ] == main.ERROR: 3525 getResults = main.FALSE 3526 sizeResponses = [] 3527 threads = [] 3528 for i in range( main.numCtrls ): 3529 t = main.Thread( target=main.CLIs[i].setTestSize, 3530 name="setTestSize-" + str( i ), 3531 args=[ onosSetName ] ) 3532 threads.append( t ) 3533 t.start() 3534 for t in threads: 3535 t.join() 3536 sizeResponses.append( t.result ) 3537 sizeResults = main.TRUE 3538 for i in range( main.numCtrls ): 3539 if size != sizeResponses[ i ]: 3540 sizeResults = main.FALSE 3541 main.log.error( "ONOS" + str( i + 1 ) + 3542 " expected a size of " + str( size ) + 3543 " for set " + onosSetName + 3544 " but got " + str( sizeResponses[ i ] ) ) 3545 addAllResults = addAllResults and getResults and sizeResults 3546 utilities.assert_equals( expect=main.TRUE, 3547 actual=addAllResults, 3548 onpass="Set addAll correct", 3549 onfail="Set addAll was incorrect" ) 3550 3551 main.step( "Distributed Set contains()" ) 3552 containsResponses = [] 3553 threads = [] 3554 for i in range( main.numCtrls ): 3555 t = main.Thread( target=main.CLIs[i].setTestGet, 3556 name="setContains-" + str( i ), 3557 args=[ onosSetName ], 3558 kwargs={ "values": addValue } ) 3559 threads.append( t ) 3560 t.start() 3561 for t in threads: 3562 t.join() 3563 # NOTE: This is the tuple 3564 containsResponses.append( t.result ) 3565 3566 containsResults = main.TRUE 3567 for i in range( main.numCtrls ): 3568 if containsResponses[ i ] == main.ERROR: 3569 containsResults = main.FALSE 3570 else: 3571 containsResults = containsResults and\ 3572 containsResponses[ i ][ 1 ] 3573 utilities.assert_equals( expect=main.TRUE, 3574 actual=containsResults, 3575 onpass="Set contains is functional", 3576 onfail="Set contains failed" ) 3577 3578 main.step( "Distributed Set containsAll()" ) 3579 containsAllResponses = [] 3580 threads = [] 3581 for i in range( main.numCtrls ): 3582 t = main.Thread( target=main.CLIs[i].setTestGet, 3583 name="setContainsAll-" + str( i ), 3584 args=[ onosSetName ], 3585 kwargs={ "values": addAllValue } ) 3586 threads.append( t ) 3587 t.start() 3588 for t in threads: 3589 t.join() 3590 # NOTE: This is the tuple 3591 containsAllResponses.append( t.result ) 3592 3593 containsAllResults = main.TRUE 3594 for i in range( main.numCtrls ): 3595 if containsResponses[ i ] == main.ERROR: 3596 containsResults = main.FALSE 3597 else: 3598 containsResults = containsResults and\ 3599 containsResponses[ i ][ 1 ] 3600 utilities.assert_equals( expect=main.TRUE, 3601 actual=containsAllResults, 3602 onpass="Set containsAll is functional", 3603 onfail="Set containsAll failed" ) 3604 3605 main.step( "Distributed Set remove()" ) 3606 onosSet.remove( addValue ) 3607 removeResponses = [] 3608 threads = [] 3609 for i in range( main.numCtrls ): 3610 t = main.Thread( target=main.CLIs[i].setTestRemove, 3611 name="setTestRemove-" + str( i ), 3612 args=[ onosSetName, addValue ] ) 3613 threads.append( t ) 3614 t.start() 3615 for t in threads: 3616 t.join() 3617 removeResponses.append( t.result ) 3618 3619 # main.TRUE = successfully changed the set 3620 # main.FALSE = action resulted in no change in set 3621 # main.ERROR - Some error in executing the function 3622 removeResults = main.TRUE 3623 for i in range( main.numCtrls ): 3624 if removeResponses[ i ] == main.TRUE: 3625 # All is well 3626 pass 3627 elif removeResponses[ i ] == main.FALSE: 3628 # not in set, probably fine 3629 pass 3630 elif removeResponses[ i ] == main.ERROR: 3631 # Error in execution 3632 removeResults = main.FALSE 3633 else: 3634 # unexpected result 3635 removeResults = main.FALSE 3636 if removeResults != main.TRUE: 3637 main.log.error( "Error executing set remove" ) 3638 3639 # Check if set is still correct 3640 size = len( onosSet ) 3641 getResponses = [] 3642 threads = [] 3643 for i in range( main.numCtrls ): 3644 t = main.Thread( target=main.CLIs[i].setTestGet, 3645 name="setTestGet-" + str( i ), 3646 args=[ onosSetName ] ) 3647 threads.append( t ) 3648 t.start() 3649 for t in threads: 3650 t.join() 3651 getResponses.append( t.result ) 3652 getResults = main.TRUE 3653 for i in range( main.numCtrls ): 3654 if isinstance( getResponses[ i ], list): 3655 current = set( getResponses[ i ] ) 3656 if len( current ) == len( getResponses[ i ] ): 3657 # no repeats 3658 if onosSet != current: 3659 main.log.error( "ONOS" + str( i + 1 ) + 3660 " has incorrect view" + 3661 " of set " + onosSetName + ":\n" + 3662 str( getResponses[ i ] ) ) 3663 main.log.debug( "Expected: " + str( onosSet ) ) 3664 main.log.debug( "Actual: " + str( current ) ) 3665 getResults = main.FALSE 3666 else: 3667 # error, set is not a set 3668 main.log.error( "ONOS" + str( i + 1 ) + 3669 " has repeat elements in" + 3670 " set " + onosSetName + ":\n" + 3671 str( getResponses[ i ] ) ) 3672 getResults = main.FALSE 3673 elif getResponses[ i ] == main.ERROR: 3674 getResults = main.FALSE 3675 sizeResponses = [] 3676 threads = [] 3677 for i in range( main.numCtrls ): 3678 t = main.Thread( target=main.CLIs[i].setTestSize, 3679 name="setTestSize-" + str( i ), 3680 args=[ onosSetName ] ) 3681 threads.append( t ) 3682 t.start() 3683 for t in threads: 3684 t.join() 3685 sizeResponses.append( t.result ) 3686 sizeResults = main.TRUE 3687 for i in range( main.numCtrls ): 3688 if size != sizeResponses[ i ]: 3689 sizeResults = main.FALSE 3690 main.log.error( "ONOS" + str( i + 1 ) + 3691 " expected a size of " + str( size ) + 3692 " for set " + onosSetName + 3693 " but got " + str( sizeResponses[ i ] ) ) 3694 removeResults = removeResults and getResults and sizeResults 3695 utilities.assert_equals( expect=main.TRUE, 3696 actual=removeResults, 3697 onpass="Set remove correct", 3698 onfail="Set remove was incorrect" ) 3699 3700 main.step( "Distributed Set removeAll()" ) 3701 onosSet.difference_update( addAllValue.split() ) 3702 removeAllResponses = [] 3703 threads = [] 3704 try: 3705 for i in range( main.numCtrls ): 3706 t = main.Thread( target=main.CLIs[i].setTestRemove, 3707 name="setTestRemoveAll-" + str( i ), 3708 args=[ onosSetName, addAllValue ] ) 3709 threads.append( t ) 3710 t.start() 3711 for t in threads: 3712 t.join() 3713 removeAllResponses.append( t.result ) 3714 except Exception, e: 3715 main.log.exception(e) 3716 3717 # main.TRUE = successfully changed the set 3718 # main.FALSE = action resulted in no change in set 3719 # main.ERROR - Some error in executing the function 3720 removeAllResults = main.TRUE 3721 for i in range( main.numCtrls ): 3722 if removeAllResponses[ i ] == main.TRUE: 3723 # All is well 3724 pass 3725 elif removeAllResponses[ i ] == main.FALSE: 3726 # not in set, probably fine 3727 pass 3728 elif removeAllResponses[ i ] == main.ERROR: 3729 # Error in execution 3730 removeAllResults = main.FALSE 3731 else: 3732 # unexpected result 3733 removeAllResults = main.FALSE 3734 if removeAllResults != main.TRUE: 3735 main.log.error( "Error executing set removeAll" ) 3736 3737 # Check if set is still correct 3738 size = len( onosSet ) 3739 getResponses = [] 3740 threads = [] 3741 for i in range( main.numCtrls ): 3742 t = main.Thread( target=main.CLIs[i].setTestGet, 3743 name="setTestGet-" + str( i ), 3744 args=[ onosSetName ] ) 3745 threads.append( t ) 3746 t.start() 3747 for t in threads: 3748 t.join() 3749 getResponses.append( t.result ) 3750 getResults = main.TRUE 3751 for i in range( main.numCtrls ): 3752 if isinstance( getResponses[ i ], list): 3753 current = set( getResponses[ i ] ) 3754 if len( current ) == len( getResponses[ i ] ): 3755 # no repeats 3756 if onosSet != current: 3757 main.log.error( "ONOS" + str( i + 1 ) + 3758 " has incorrect view" + 3759 " of set " + onosSetName + ":\n" + 3760 str( getResponses[ i ] ) ) 3761 main.log.debug( "Expected: " + str( onosSet ) ) 3762 main.log.debug( "Actual: " + str( current ) ) 3763 getResults = main.FALSE 3764 else: 3765 # error, set is not a set 3766 main.log.error( "ONOS" + str( i + 1 ) + 3767 " has repeat elements in" + 3768 " set " + onosSetName + ":\n" + 3769 str( getResponses[ i ] ) ) 3770 getResults = main.FALSE 3771 elif getResponses[ i ] == main.ERROR: 3772 getResults = main.FALSE 3773 sizeResponses = [] 3774 threads = [] 3775 for i in range( main.numCtrls ): 3776 t = main.Thread( target=main.CLIs[i].setTestSize, 3777 name="setTestSize-" + str( i ), 3778 args=[ onosSetName ] ) 3779 threads.append( t ) 3780 t.start() 3781 for t in threads: 3782 t.join() 3783 sizeResponses.append( t.result ) 3784 sizeResults = main.TRUE 3785 for i in range( main.numCtrls ): 3786 if size != sizeResponses[ i ]: 3787 sizeResults = main.FALSE 3788 main.log.error( "ONOS" + str( i + 1 ) + 3789 " expected a size of " + str( size ) + 3790 " for set " + onosSetName + 3791 " but got " + str( sizeResponses[ i ] ) ) 3792 removeAllResults = removeAllResults and getResults and sizeResults 3793 utilities.assert_equals( expect=main.TRUE, 3794 actual=removeAllResults, 3795 onpass="Set removeAll correct", 3796 onfail="Set removeAll was incorrect" ) 3797 3798 main.step( "Distributed Set addAll()" ) 3799 onosSet.update( addAllValue.split() ) 3800 addResponses = [] 3801 threads = [] 3802 for i in range( main.numCtrls ): 3803 t = main.Thread( target=main.CLIs[i].setTestAdd, 3804 name="setTestAddAll-" + str( i ), 3805 args=[ onosSetName, addAllValue ] ) 3806 threads.append( t ) 3807 t.start() 3808 for t in threads: 3809 t.join() 3810 addResponses.append( t.result ) 3811 3812 # main.TRUE = successfully changed the set 3813 # main.FALSE = action resulted in no change in set 3814 # main.ERROR - Some error in executing the function 3815 addAllResults = main.TRUE 3816 for i in range( main.numCtrls ): 3817 if addResponses[ i ] == main.TRUE: 3818 # All is well 3819 pass 3820 elif addResponses[ i ] == main.FALSE: 3821 # Already in set, probably fine 3822 pass 3823 elif addResponses[ i ] == main.ERROR: 3824 # Error in execution 3825 addAllResults = main.FALSE 3826 else: 3827 # unexpected result 3828 addAllResults = main.FALSE 3829 if addAllResults != main.TRUE: 3830 main.log.error( "Error executing set addAll" ) 3831 3832 # Check if set is still correct 3833 size = len( onosSet ) 3834 getResponses = [] 3835 threads = [] 3836 for i in range( main.numCtrls ): 3837 t = main.Thread( target=main.CLIs[i].setTestGet, 3838 name="setTestGet-" + str( i ), 3839 args=[ onosSetName ] ) 3840 threads.append( t ) 3841 t.start() 3842 for t in threads: 3843 t.join() 3844 getResponses.append( t.result ) 3845 getResults = main.TRUE 3846 for i in range( main.numCtrls ): 3847 if isinstance( getResponses[ i ], list): 3848 current = set( getResponses[ i ] ) 3849 if len( current ) == len( getResponses[ i ] ): 3850 # no repeats 3851 if onosSet != current: 3852 main.log.error( "ONOS" + str( i + 1 ) + 3853 " has incorrect view" + 3854 " of set " + onosSetName + ":\n" + 3855 str( getResponses[ i ] ) ) 3856 main.log.debug( "Expected: " + str( onosSet ) ) 3857 main.log.debug( "Actual: " + str( current ) ) 3858 getResults = main.FALSE 3859 else: 3860 # error, set is not a set 3861 main.log.error( "ONOS" + str( i + 1 ) + 3862 " has repeat elements in" + 3863 " set " + onosSetName + ":\n" + 3864 str( getResponses[ i ] ) ) 3865 getResults = main.FALSE 3866 elif getResponses[ i ] == main.ERROR: 3867 getResults = main.FALSE 3868 sizeResponses = [] 3869 threads = [] 3870 for i in range( main.numCtrls ): 3871 t = main.Thread( target=main.CLIs[i].setTestSize, 3872 name="setTestSize-" + str( i ), 3873 args=[ onosSetName ] ) 3874 threads.append( t ) 3875 t.start() 3876 for t in threads: 3877 t.join() 3878 sizeResponses.append( t.result ) 3879 sizeResults = main.TRUE 3880 for i in range( main.numCtrls ): 3881 if size != sizeResponses[ i ]: 3882 sizeResults = main.FALSE 3883 main.log.error( "ONOS" + str( i + 1 ) + 3884 " expected a size of " + str( size ) + 3885 " for set " + onosSetName + 3886 " but got " + str( sizeResponses[ i ] ) ) 3887 addAllResults = addAllResults and getResults and sizeResults 3888 utilities.assert_equals( expect=main.TRUE, 3889 actual=addAllResults, 3890 onpass="Set addAll correct", 3891 onfail="Set addAll was incorrect" ) 3892 3893 main.step( "Distributed Set clear()" ) 3894 onosSet.clear() 3895 clearResponses = [] 3896 threads = [] 3897 for i in range( main.numCtrls ): 3898 t = main.Thread( target=main.CLIs[i].setTestRemove, 3899 name="setTestClear-" + str( i ), 3900 args=[ onosSetName, " "], # Values doesn't matter 3901 kwargs={ "clear": True } ) 3902 threads.append( t ) 3903 t.start() 3904 for t in threads: 3905 t.join() 3906 clearResponses.append( t.result ) 3907 3908 # main.TRUE = successfully changed the set 3909 # main.FALSE = action resulted in no change in set 3910 # main.ERROR - Some error in executing the function 3911 clearResults = main.TRUE 3912 for i in range( main.numCtrls ): 3913 if clearResponses[ i ] == main.TRUE: 3914 # All is well 3915 pass 3916 elif clearResponses[ i ] == main.FALSE: 3917 # Nothing set, probably fine 3918 pass 3919 elif clearResponses[ i ] == main.ERROR: 3920 # Error in execution 3921 clearResults = main.FALSE 3922 else: 3923 # unexpected result 3924 clearResults = main.FALSE 3925 if clearResults != main.TRUE: 3926 main.log.error( "Error executing set clear" ) 3927 3928 # Check if set is still correct 3929 size = len( onosSet ) 3930 getResponses = [] 3931 threads = [] 3932 for i in range( main.numCtrls ): 3933 t = main.Thread( target=main.CLIs[i].setTestGet, 3934 name="setTestGet-" + str( i ), 3935 args=[ onosSetName ] ) 3936 threads.append( t ) 3937 t.start() 3938 for t in threads: 3939 t.join() 3940 getResponses.append( t.result ) 3941 getResults = main.TRUE 3942 for i in range( main.numCtrls ): 3943 if isinstance( getResponses[ i ], list): 3944 current = set( getResponses[ i ] ) 3945 if len( current ) == len( getResponses[ i ] ): 3946 # no repeats 3947 if onosSet != current: 3948 main.log.error( "ONOS" + str( i + 1 ) + 3949 " has incorrect view" + 3950 " of set " + onosSetName + ":\n" + 3951 str( getResponses[ i ] ) ) 3952 main.log.debug( "Expected: " + str( onosSet ) ) 3953 main.log.debug( "Actual: " + str( current ) ) 3954 getResults = main.FALSE 3955 else: 3956 # error, set is not a set 3957 main.log.error( "ONOS" + str( i + 1 ) + 3958 " has repeat elements in" + 3959 " set " + onosSetName + ":\n" + 3960 str( getResponses[ i ] ) ) 3961 getResults = main.FALSE 3962 elif getResponses[ i ] == main.ERROR: 3963 getResults = main.FALSE 3964 sizeResponses = [] 3965 threads = [] 3966 for i in range( main.numCtrls ): 3967 t = main.Thread( target=main.CLIs[i].setTestSize, 3968 name="setTestSize-" + str( i ), 3969 args=[ onosSetName ] ) 3970 threads.append( t ) 3971 t.start() 3972 for t in threads: 3973 t.join() 3974 sizeResponses.append( t.result ) 3975 sizeResults = main.TRUE 3976 for i in range( main.numCtrls ): 3977 if size != sizeResponses[ i ]: 3978 sizeResults = main.FALSE 3979 main.log.error( "ONOS" + str( i + 1 ) + 3980 " expected a size of " + str( size ) + 3981 " for set " + onosSetName + 3982 " but got " + str( sizeResponses[ i ] ) ) 3983 clearResults = clearResults and getResults and sizeResults 3984 utilities.assert_equals( expect=main.TRUE, 3985 actual=clearResults, 3986 onpass="Set clear correct", 3987 onfail="Set clear was incorrect" ) 3988 3989 main.step( "Distributed Set addAll()" ) 3990 onosSet.update( addAllValue.split() ) 3991 addResponses = [] 3992 threads = [] 3993 for i in range( main.numCtrls ): 3994 t = main.Thread( target=main.CLIs[i].setTestAdd, 3995 name="setTestAddAll-" + str( i ), 3996 args=[ onosSetName, addAllValue ] ) 3997 threads.append( t ) 3998 t.start() 3999 for t in threads: 4000 t.join() 4001 addResponses.append( t.result ) 4002 4003 # main.TRUE = successfully changed the set 4004 # main.FALSE = action resulted in no change in set 4005 # main.ERROR - Some error in executing the function 4006 addAllResults = main.TRUE 4007 for i in range( main.numCtrls ): 4008 if addResponses[ i ] == main.TRUE: 4009 # All is well 4010 pass 4011 elif addResponses[ i ] == main.FALSE: 4012 # Already in set, probably fine 4013 pass 4014 elif addResponses[ i ] == main.ERROR: 4015 # Error in execution 4016 addAllResults = main.FALSE 4017 else: 4018 # unexpected result 4019 addAllResults = main.FALSE 4020 if addAllResults != main.TRUE: 4021 main.log.error( "Error executing set addAll" ) 4022 4023 # Check if set is still correct 4024 size = len( onosSet ) 4025 getResponses = [] 4026 threads = [] 4027 for i in range( main.numCtrls ): 4028 t = main.Thread( target=main.CLIs[i].setTestGet, 4029 name="setTestGet-" + str( i ), 4030 args=[ onosSetName ] ) 4031 threads.append( t ) 4032 t.start() 4033 for t in threads: 4034 t.join() 4035 getResponses.append( t.result ) 4036 getResults = main.TRUE 4037 for i in range( main.numCtrls ): 4038 if isinstance( getResponses[ i ], list): 4039 current = set( getResponses[ i ] ) 4040 if len( current ) == len( getResponses[ i ] ): 4041 # no repeats 4042 if onosSet != current: 4043 main.log.error( "ONOS" + str( i + 1 ) + 4044 " has incorrect view" + 4045 " of set " + onosSetName + ":\n" + 4046 str( getResponses[ i ] ) ) 4047 main.log.debug( "Expected: " + str( onosSet ) ) 4048 main.log.debug( "Actual: " + str( current ) ) 4049 getResults = main.FALSE 4050 else: 4051 # error, set is not a set 4052 main.log.error( "ONOS" + str( i + 1 ) + 4053 " has repeat elements in" + 4054 " set " + onosSetName + ":\n" + 4055 str( getResponses[ i ] ) ) 4056 getResults = main.FALSE 4057 elif getResponses[ i ] == main.ERROR: 4058 getResults = main.FALSE 4059 sizeResponses = [] 4060 threads = [] 4061 for i in range( main.numCtrls ): 4062 t = main.Thread( target=main.CLIs[i].setTestSize, 4063 name="setTestSize-" + str( i ), 4064 args=[ onosSetName ] ) 4065 threads.append( t ) 4066 t.start() 4067 for t in threads: 4068 t.join() 4069 sizeResponses.append( t.result ) 4070 sizeResults = main.TRUE 4071 for i in range( main.numCtrls ): 4072 if size != sizeResponses[ i ]: 4073 sizeResults = main.FALSE 4074 main.log.error( "ONOS" + str( i + 1 ) + 4075 " expected a size of " + str( size ) + 4076 " for set " + onosSetName + 4077 " but got " + str( sizeResponses[ i ] ) ) 4078 addAllResults = addAllResults and getResults and sizeResults 4079 utilities.assert_equals( expect=main.TRUE, 4080 actual=addAllResults, 4081 onpass="Set addAll correct", 4082 onfail="Set addAll was incorrect" ) 4083 4084 main.step( "Distributed Set retain()" ) 4085 onosSet.intersection_update( retainValue.split() ) 4086 retainResponses = [] 4087 threads = [] 4088 for i in range( main.numCtrls ): 4089 t = main.Thread( target=main.CLIs[i].setTestRemove, 4090 name="setTestRetain-" + str( i ), 4091 args=[ onosSetName, retainValue ], 4092 kwargs={ "retain": True } ) 4093 threads.append( t ) 4094 t.start() 4095 for t in threads: 4096 t.join() 4097 retainResponses.append( t.result ) 4098 4099 # main.TRUE = successfully changed the set 4100 # main.FALSE = action resulted in no change in set 4101 # main.ERROR - Some error in executing the function 4102 retainResults = main.TRUE 4103 for i in range( main.numCtrls ): 4104 if retainResponses[ i ] == main.TRUE: 4105 # All is well 4106 pass 4107 elif retainResponses[ i ] == main.FALSE: 4108 # Already in set, probably fine 4109 pass 4110 elif retainResponses[ i ] == main.ERROR: 4111 # Error in execution 4112 retainResults = main.FALSE 4113 else: 4114 # unexpected result 4115 retainResults = main.FALSE 4116 if retainResults != main.TRUE: 4117 main.log.error( "Error executing set retain" ) 4118 4119 # Check if set is still correct 4120 size = len( onosSet ) 4121 getResponses = [] 4122 threads = [] 4123 for i in range( main.numCtrls ): 4124 t = main.Thread( target=main.CLIs[i].setTestGet, 4125 name="setTestGet-" + str( i ), 4126 args=[ onosSetName ] ) 4127 threads.append( t ) 4128 t.start() 4129 for t in threads: 4130 t.join() 4131 getResponses.append( t.result ) 4132 getResults = main.TRUE 4133 for i in range( main.numCtrls ): 4134 if isinstance( getResponses[ i ], list): 4135 current = set( getResponses[ i ] ) 4136 if len( current ) == len( getResponses[ i ] ): 4137 # no repeats 4138 if onosSet != current: 4139 main.log.error( "ONOS" + str( i + 1 ) + 4140 " has incorrect view" + 4141 " of set " + onosSetName + ":\n" + 4142 str( getResponses[ i ] ) ) 4143 main.log.debug( "Expected: " + str( onosSet ) ) 4144 main.log.debug( "Actual: " + str( current ) ) 4145 getResults = main.FALSE 4146 else: 4147 # error, set is not a set 4148 main.log.error( "ONOS" + str( i + 1 ) + 4149 " has repeat elements in" + 4150 " set " + onosSetName + ":\n" + 4151 str( getResponses[ i ] ) ) 4152 getResults = main.FALSE 4153 elif getResponses[ i ] == main.ERROR: 4154 getResults = main.FALSE 4155 sizeResponses = [] 4156 threads = [] 4157 for i in range( main.numCtrls ): 4158 t = main.Thread( target=main.CLIs[i].setTestSize, 4159 name="setTestSize-" + str( i ), 4160 args=[ onosSetName ] ) 4161 threads.append( t ) 4162 t.start() 4163 for t in threads: 4164 t.join() 4165 sizeResponses.append( t.result ) 4166 sizeResults = main.TRUE 4167 for i in range( main.numCtrls ): 4168 if size != sizeResponses[ i ]: 4169 sizeResults = main.FALSE 4170 main.log.error( "ONOS" + str( i + 1 ) + 4171 " expected a size of " + 4172 str( size ) + " for set " + onosSetName + 4173 " but got " + str( sizeResponses[ i ] ) ) 4174 retainResults = retainResults and getResults and sizeResults 4175 utilities.assert_equals( expect=main.TRUE, 4176 actual=retainResults, 4177 onpass="Set retain correct", 4178 onfail="Set retain was incorrect" )
4179