Package TestON :: Package tests :: Package HAminorityRestart :: Module HAminorityRestart
[hide private]
[frames] | no frames]

Source Code for Module TestON.tests.HAminorityRestart.HAminorityRestart

   1  """ 
   2  Description: This test is to determine if ONOS can handle 
   3      a minority of it's nodes restarting 
   4   
   5  List of test cases: 
   6  CASE1: Compile ONOS and push it to the test machines 
   7  CASE2: Assign devices to controllers 
   8  CASE21: Assign mastership to controllers 
   9  CASE3: Assign intents 
  10  CASE4: Ping across added host intents 
  11  CASE5: Reading state of ONOS 
  12  CASE6: The Failure case. 
  13  CASE7: Check state after control plane failure 
  14  CASE8: Compare topo 
  15  CASE9: Link s3-s28 down 
  16  CASE10: Link s3-s28 up 
  17  CASE11: Switch down 
  18  CASE12: Switch up 
  19  CASE13: Clean up 
  20  CASE14: start election app on all onos nodes 
  21  CASE15: Check that Leadership Election is still functional 
  22  CASE16: Install Distributed Primitives app 
  23  CASE17: Check for basic functionality with distributed primitives 
  24  """ 
  25   
  26   
27 -class HAminorityRestart:
28
29 - def __init__( self ):
30 self.default = ''
31
32 - def CASE1( self, main ):
33 """ 34 CASE1 is to compile ONOS and push it to the test machines 35 36 Startup sequence: 37 cell <name> 38 onos-verify-cell 39 NOTE: temporary - onos-remove-raft-logs 40 onos-uninstall 41 start mininet 42 git pull 43 mvn clean install 44 onos-package 45 onos-install -f 46 onos-wait-for-start 47 start cli sessions 48 start tcpdump 49 """ 50 import imp 51 main.log.info( "ONOS HA test: Restart minority of ONOS nodes - " + 52 "initialization" ) 53 main.case( "Setting up test environment" ) 54 main.caseExplanation = "Setup the test environment including " +\ 55 "installing ONOS, starting Mininet and ONOS" +\ 56 "cli sessions." 57 # TODO: save all the timers and output them for plotting 58 59 # load some variables from the params file 60 PULLCODE = False 61 if main.params[ 'Git' ] == 'True': 62 PULLCODE = True 63 gitBranch = main.params[ 'branch' ] 64 cellName = main.params[ 'ENV' ][ 'cellName' ] 65 66 main.numCtrls = int( main.params[ 'num_controllers' ] ) 67 if main.ONOSbench.maxNodes: 68 if main.ONOSbench.maxNodes < main.numCtrls: 69 main.numCtrls = int( main.ONOSbench.maxNodes ) 70 # set global variables 71 global ONOS1Port 72 global ONOS2Port 73 global ONOS3Port 74 global ONOS4Port 75 global ONOS5Port 76 global ONOS6Port 77 global ONOS7Port 78 79 # FIXME: just get controller port from params? 80 # TODO: do we really need all these? 81 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ] 82 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ] 83 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ] 84 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ] 85 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ] 86 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ] 87 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ] 88 89 try: 90 fileName = "Counters" 91 # TODO: Maybe make a library folder somewhere? 92 path = main.params[ 'imports' ][ 'path' ] 93 main.Counters = imp.load_source( fileName, 94 path + fileName + ".py" ) 95 except Exception as e: 96 main.log.exception( e ) 97 main.cleanup() 98 main.exit() 99 100 main.CLIs = [] 101 main.nodes = [] 102 ipList = [] 103 for i in range( 1, main.numCtrls + 1 ): 104 try: 105 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) ) 106 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) ) 107 ipList.append( main.nodes[ -1 ].ip_address ) 108 except AttributeError: 109 break 110 111 main.step( "Create cell file" ) 112 cellAppString = main.params[ 'ENV' ][ 'appString' ] 113 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName, 114 main.Mininet1.ip_address, 115 cellAppString, ipList ) 116 main.step( "Applying cell variable to environment" ) 117 cellResult = main.ONOSbench.setCell( cellName ) 118 verifyResult = main.ONOSbench.verifyCell() 119 120 # FIXME:this is short term fix 121 main.log.info( "Removing raft logs" ) 122 main.ONOSbench.onosRemoveRaftLogs() 123 124 main.log.info( "Uninstalling ONOS" ) 125 for node in main.nodes: 126 main.ONOSbench.onosUninstall( node.ip_address ) 127 128 # Make sure ONOS is DEAD 129 main.log.info( "Killing any ONOS processes" ) 130 killResults = main.TRUE 131 for node in main.nodes: 132 killed = main.ONOSbench.onosKill( node.ip_address ) 133 killResults = killResults and killed 134 135 cleanInstallResult = main.TRUE 136 gitPullResult = main.TRUE 137 138 main.step( "Starting Mininet" ) 139 # scp topo file to mininet 140 # TODO: move to params? 141 topoName = "obelisk.py" 142 filePath = main.ONOSbench.home + "/tools/test/topos/" 143 main.ONOSbench.copyMininetFile( topoName, filePath, 144 main.Mininet1.user_name, 145 main.Mininet1.ip_address ) 146 mnResult = main.Mininet1.startNet( ) 147 utilities.assert_equals( expect=main.TRUE, actual=mnResult, 148 onpass="Mininet Started", 149 onfail="Error starting Mininet" ) 150 151 main.step( "Git checkout and pull " + gitBranch ) 152 if PULLCODE: 153 main.ONOSbench.gitCheckout( gitBranch ) 154 gitPullResult = main.ONOSbench.gitPull() 155 # values of 1 or 3 are good 156 utilities.assert_lesser( expect=0, actual=gitPullResult, 157 onpass="Git pull successful", 158 onfail="Git pull failed" ) 159 main.ONOSbench.getVersion( report=True ) 160 161 main.step( "Using mvn clean install" ) 162 cleanInstallResult = main.TRUE 163 if PULLCODE and gitPullResult == main.TRUE: 164 cleanInstallResult = main.ONOSbench.cleanInstall() 165 else: 166 main.log.warn( "Did not pull new code so skipping mvn " + 167 "clean install" ) 168 utilities.assert_equals( expect=main.TRUE, 169 actual=cleanInstallResult, 170 onpass="MCI successful", 171 onfail="MCI failed" ) 172 # GRAPHS 173 # NOTE: important params here: 174 # job = name of Jenkins job 175 # Plot Name = Plot-HA, only can be used if multiple plots 176 # index = The number of the graph under plot name 177 job = "HAminorityRestart" 178 plotName = "Plot-HA" 179 graphs = '<ac:structured-macro ac:name="html">\n' 180 graphs += '<ac:plain-text-body><![CDATA[\n' 181 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\ 182 '/plot/' + plotName + '/getPlot?index=0' +\ 183 '&width=500&height=300"' +\ 184 'noborder="0" width="500" height="300" scrolling="yes" ' +\ 185 'seamless="seamless"></iframe>\n' 186 graphs += ']]></ac:plain-text-body>\n' 187 graphs += '</ac:structured-macro>\n' 188 main.log.wiki(graphs) 189 190 main.step( "Creating ONOS package" ) 191 packageResult = main.ONOSbench.onosPackage() 192 utilities.assert_equals( expect=main.TRUE, actual=packageResult, 193 onpass="ONOS package successful", 194 onfail="ONOS package failed" ) 195 196 main.step( "Installing ONOS package" ) 197 onosInstallResult = main.TRUE 198 for node in main.nodes: 199 tmpResult = main.ONOSbench.onosInstall( options="-f", 200 node=node.ip_address ) 201 onosInstallResult = onosInstallResult and tmpResult 202 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult, 203 onpass="ONOS install successful", 204 onfail="ONOS install failed" ) 205 206 main.step( "Checking if ONOS is up yet" ) 207 for i in range( 2 ): 208 onosIsupResult = main.TRUE 209 for node in main.nodes: 210 started = main.ONOSbench.isup( node.ip_address ) 211 if not started: 212 main.log.error( node.name + " didn't start!" ) 213 main.ONOSbench.onosStop( node.ip_address ) 214 main.ONOSbench.onosStart( node.ip_address ) 215 onosIsupResult = onosIsupResult and started 216 if onosIsupResult == main.TRUE: 217 break 218 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult, 219 onpass="ONOS startup successful", 220 onfail="ONOS startup failed" ) 221 222 main.log.step( "Starting ONOS CLI sessions" ) 223 cliResults = main.TRUE 224 threads = [] 225 for i in range( main.numCtrls ): 226 t = main.Thread( target=main.CLIs[i].startOnosCli, 227 name="startOnosCli-" + str( i ), 228 args=[main.nodes[i].ip_address] ) 229 threads.append( t ) 230 t.start() 231 232 for t in threads: 233 t.join() 234 cliResults = cliResults and t.result 235 utilities.assert_equals( expect=main.TRUE, actual=cliResults, 236 onpass="ONOS cli startup successful", 237 onfail="ONOS cli startup failed" ) 238 239 if main.params[ 'tcpdump' ].lower() == "true": 240 main.step( "Start Packet Capture MN" ) 241 main.Mininet2.startTcpdump( 242 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST ) 243 + "-MN.pcap", 244 intf=main.params[ 'MNtcpdump' ][ 'intf' ], 245 port=main.params[ 'MNtcpdump' ][ 'port' ] ) 246 247 main.step( "App Ids check" ) 248 appCheck = main.TRUE 249 threads = [] 250 for i in range( main.numCtrls ): 251 t = main.Thread( target=main.CLIs[i].appToIDCheck, 252 name="appToIDCheck-" + str( i ), 253 args=[] ) 254 threads.append( t ) 255 t.start() 256 257 for t in threads: 258 t.join() 259 appCheck = appCheck and t.result 260 if appCheck != main.TRUE: 261 main.log.warn( main.CLIs[0].apps() ) 262 main.log.warn( main.CLIs[0].appIDs() ) 263 utilities.assert_equals( expect=main.TRUE, actual=appCheck, 264 onpass="App Ids seem to be correct", 265 onfail="Something is wrong with app Ids" ) 266 267 if cliResults == main.FALSE: 268 main.log.error( "Failed to start ONOS, stopping test" ) 269 main.cleanup() 270 main.exit()
271
272 - def CASE2( self, main ):
273 """ 274 Assign devices to controllers 275 """ 276 import re 277 assert main.numCtrls, "main.numCtrls not defined" 278 assert main, "main not defined" 279 assert utilities.assert_equals, "utilities.assert_equals not defined" 280 assert main.CLIs, "main.CLIs not defined" 281 assert main.nodes, "main.nodes not defined" 282 assert ONOS1Port, "ONOS1Port not defined" 283 assert ONOS2Port, "ONOS2Port not defined" 284 assert ONOS3Port, "ONOS3Port not defined" 285 assert ONOS4Port, "ONOS4Port not defined" 286 assert ONOS5Port, "ONOS5Port not defined" 287 assert ONOS6Port, "ONOS6Port not defined" 288 assert ONOS7Port, "ONOS7Port not defined" 289 290 main.case( "Assigning devices to controllers" ) 291 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\ 292 "and check that an ONOS node becomes the " +\ 293 "master of the device." 294 main.step( "Assign switches to controllers" ) 295 296 ipList = [] 297 for i in range( main.numCtrls ): 298 ipList.append( main.nodes[ i ].ip_address ) 299 swList = [] 300 for i in range( 1, 29 ): 301 swList.append( "s" + str( i ) ) 302 main.Mininet1.assignSwController( sw=swList, ip=ipList ) 303 304 mastershipCheck = main.TRUE 305 for i in range( 1, 29 ): 306 response = main.Mininet1.getSwController( "s" + str( i ) ) 307 try: 308 main.log.info( str( response ) ) 309 except Exception: 310 main.log.info( repr( response ) ) 311 for node in main.nodes: 312 if re.search( "tcp:" + node.ip_address, response ): 313 mastershipCheck = mastershipCheck and main.TRUE 314 else: 315 main.log.error( "Error, node " + node.ip_address + " is " + 316 "not in the list of controllers s" + 317 str( i ) + " is connecting to." ) 318 mastershipCheck = main.FALSE 319 utilities.assert_equals( 320 expect=main.TRUE, 321 actual=mastershipCheck, 322 onpass="Switch mastership assigned correctly", 323 onfail="Switches not assigned correctly to controllers" )
324
325 - def CASE21( self, main ):
326 """ 327 Assign mastership to controllers 328 """ 329 import time 330 assert main.numCtrls, "main.numCtrls not defined" 331 assert main, "main not defined" 332 assert utilities.assert_equals, "utilities.assert_equals not defined" 333 assert main.CLIs, "main.CLIs not defined" 334 assert main.nodes, "main.nodes not defined" 335 assert ONOS1Port, "ONOS1Port not defined" 336 assert ONOS2Port, "ONOS2Port not defined" 337 assert ONOS3Port, "ONOS3Port not defined" 338 assert ONOS4Port, "ONOS4Port not defined" 339 assert ONOS5Port, "ONOS5Port not defined" 340 assert ONOS6Port, "ONOS6Port not defined" 341 assert ONOS7Port, "ONOS7Port not defined" 342 343 main.case( "Assigning Controller roles for switches" ) 344 main.caseExplanation = "Check that ONOS is connected to each " +\ 345 "device. Then manually assign" +\ 346 " mastership to specific ONOS nodes using" +\ 347 " 'device-role'" 348 main.step( "Assign mastership of switches to specific controllers" ) 349 # Manually assign mastership to the controller we want 350 roleCall = main.TRUE 351 352 ipList = [ ] 353 deviceList = [] 354 try: 355 # Assign mastership to specific controllers. This assignment was 356 # determined for a 7 node cluser, but will work with any sized 357 # cluster 358 for i in range( 1, 29 ): # switches 1 through 28 359 # set up correct variables: 360 if i == 1: 361 c = 0 362 ip = main.nodes[ c ].ip_address # ONOS1 363 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' ) 364 elif i == 2: 365 c = 1 % main.numCtrls 366 ip = main.nodes[ c ].ip_address # ONOS2 367 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' ) 368 elif i == 3: 369 c = 1 % main.numCtrls 370 ip = main.nodes[ c ].ip_address # ONOS2 371 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' ) 372 elif i == 4: 373 c = 3 % main.numCtrls 374 ip = main.nodes[ c ].ip_address # ONOS4 375 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' ) 376 elif i == 5: 377 c = 2 % main.numCtrls 378 ip = main.nodes[ c ].ip_address # ONOS3 379 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' ) 380 elif i == 6: 381 c = 2 % main.numCtrls 382 ip = main.nodes[ c ].ip_address # ONOS3 383 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' ) 384 elif i == 7: 385 c = 5 % main.numCtrls 386 ip = main.nodes[ c ].ip_address # ONOS6 387 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' ) 388 elif i >= 8 and i <= 17: 389 c = 4 % main.numCtrls 390 ip = main.nodes[ c ].ip_address # ONOS5 391 dpid = '3' + str( i ).zfill( 3 ) 392 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' ) 393 elif i >= 18 and i <= 27: 394 c = 6 % main.numCtrls 395 ip = main.nodes[ c ].ip_address # ONOS7 396 dpid = '6' + str( i ).zfill( 3 ) 397 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' ) 398 elif i == 28: 399 c = 0 400 ip = main.nodes[ c ].ip_address # ONOS1 401 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' ) 402 else: 403 main.log.error( "You didn't write an else statement for " + 404 "switch s" + str( i ) ) 405 roleCall = main.FALSE 406 # Assign switch 407 assert deviceId, "No device id for s" + str( i ) + " in ONOS" 408 # TODO: make this controller dynamic 409 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId, 410 ip ) 411 ipList.append( ip ) 412 deviceList.append( deviceId ) 413 except ( AttributeError, AssertionError ): 414 main.log.exception( "Something is wrong with ONOS device view" ) 415 main.log.info( main.ONOScli1.devices() ) 416 utilities.assert_equals( 417 expect=main.TRUE, 418 actual=roleCall, 419 onpass="Re-assigned switch mastership to designated controller", 420 onfail="Something wrong with deviceRole calls" ) 421 422 main.step( "Check mastership was correctly assigned" ) 423 roleCheck = main.TRUE 424 # NOTE: This is due to the fact that device mastership change is not 425 # atomic and is actually a multi step process 426 time.sleep( 5 ) 427 for i in range( len( ipList ) ): 428 ip = ipList[i] 429 deviceId = deviceList[i] 430 # Check assignment 431 master = main.ONOScli1.getRole( deviceId ).get( 'master' ) 432 if ip in master: 433 roleCheck = roleCheck and main.TRUE 434 else: 435 roleCheck = roleCheck and main.FALSE 436 main.log.error( "Error, controller " + ip + " is not" + 437 " master " + "of device " + 438 str( deviceId ) + ". Master is " + 439 repr( master ) + "." ) 440 utilities.assert_equals( 441 expect=main.TRUE, 442 actual=roleCheck, 443 onpass="Switches were successfully reassigned to designated " + 444 "controller", 445 onfail="Switches were not successfully reassigned" )
446
447 - def CASE3( self, main ):
448 """ 449 Assign intents 450 """ 451 import time 452 import json 453 assert main.numCtrls, "main.numCtrls not defined" 454 assert main, "main not defined" 455 assert utilities.assert_equals, "utilities.assert_equals not defined" 456 assert main.CLIs, "main.CLIs not defined" 457 assert main.nodes, "main.nodes not defined" 458 main.case( "Adding host Intents" ) 459 main.caseExplanation = "Discover hosts by using pingall then " +\ 460 "assign predetermined host-to-host intents." +\ 461 " After installation, check that the intent" +\ 462 " is distributed to all nodes and the state" +\ 463 " is INSTALLED" 464 465 # install onos-app-fwd 466 main.step( "Install reactive forwarding app" ) 467 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" ) 468 utilities.assert_equals( expect=main.TRUE, actual=installResults, 469 onpass="Install fwd successful", 470 onfail="Install fwd failed" ) 471 472 main.step( "Check app ids" ) 473 appCheck = main.TRUE 474 threads = [] 475 for i in range( main.numCtrls ): 476 t = main.Thread( target=main.CLIs[i].appToIDCheck, 477 name="appToIDCheck-" + str( i ), 478 args=[] ) 479 threads.append( t ) 480 t.start() 481 482 for t in threads: 483 t.join() 484 appCheck = appCheck and t.result 485 if appCheck != main.TRUE: 486 main.log.warn( main.CLIs[0].apps() ) 487 main.log.warn( main.CLIs[0].appIDs() ) 488 utilities.assert_equals( expect=main.TRUE, actual=appCheck, 489 onpass="App Ids seem to be correct", 490 onfail="Something is wrong with app Ids" ) 491 492 main.step( "Discovering Hosts( Via pingall for now )" ) 493 # FIXME: Once we have a host discovery mechanism, use that instead 494 # REACTIVE FWD test 495 pingResult = main.FALSE 496 for i in range(2): # Retry if pingall fails first time 497 time1 = time.time() 498 pingResult = main.Mininet1.pingall() 499 if i == 0: 500 utilities.assert_equals( 501 expect=main.TRUE, 502 actual=pingResult, 503 onpass="Reactive Pingall test passed", 504 onfail="Reactive Pingall failed, " + 505 "one or more ping pairs failed" ) 506 time2 = time.time() 507 main.log.info( "Time for pingall: %2f seconds" % 508 ( time2 - time1 ) ) 509 # timeout for fwd flows 510 time.sleep( 11 ) 511 # uninstall onos-app-fwd 512 main.step( "Uninstall reactive forwarding app" ) 513 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" ) 514 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult, 515 onpass="Uninstall fwd successful", 516 onfail="Uninstall fwd failed" ) 517 518 main.step( "Check app ids" ) 519 threads = [] 520 appCheck2 = main.TRUE 521 for i in range( main.numCtrls ): 522 t = main.Thread( target=main.CLIs[i].appToIDCheck, 523 name="appToIDCheck-" + str( i ), 524 args=[] ) 525 threads.append( t ) 526 t.start() 527 528 for t in threads: 529 t.join() 530 appCheck2 = appCheck2 and t.result 531 if appCheck2 != main.TRUE: 532 main.log.warn( main.CLIs[0].apps() ) 533 main.log.warn( main.CLIs[0].appIDs() ) 534 utilities.assert_equals( expect=main.TRUE, actual=appCheck2, 535 onpass="App Ids seem to be correct", 536 onfail="Something is wrong with app Ids" ) 537 538 main.step( "Add host intents via cli" ) 539 intentIds = [] 540 # TODO: move the host numbers to params 541 # Maybe look at all the paths we ping? 542 intentAddResult = True 543 hostResult = main.TRUE 544 for i in range( 8, 18 ): 545 main.log.info( "Adding host intent between h" + str( i ) + 546 " and h" + str( i + 10 ) ) 547 host1 = "00:00:00:00:00:" + \ 548 str( hex( i )[ 2: ] ).zfill( 2 ).upper() 549 host2 = "00:00:00:00:00:" + \ 550 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper() 551 # NOTE: getHost can return None 552 host1Dict = main.ONOScli1.getHost( host1 ) 553 host2Dict = main.ONOScli1.getHost( host2 ) 554 host1Id = None 555 host2Id = None 556 if host1Dict and host2Dict: 557 host1Id = host1Dict.get( 'id', None ) 558 host2Id = host2Dict.get( 'id', None ) 559 if host1Id and host2Id: 560 nodeNum = ( i % main.numCtrls ) 561 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id ) 562 if tmpId: 563 main.log.info( "Added intent with id: " + tmpId ) 564 intentIds.append( tmpId ) 565 else: 566 main.log.error( "addHostIntent returned: " + 567 repr( tmpId ) ) 568 else: 569 main.log.error( "Error, getHost() failed for h" + str( i ) + 570 " and/or h" + str( i + 10 ) ) 571 hosts = main.CLIs[ 0 ].hosts() 572 main.log.warn( "Hosts output: " ) 573 try: 574 main.log.warn( json.dumps( json.loads( hosts ), 575 sort_keys=True, 576 indent=4, 577 separators=( ',', ': ' ) ) ) 578 except ( ValueError, TypeError ): 579 main.log.warn( repr( hosts ) ) 580 hostResult = main.FALSE 581 utilities.assert_equals( expect=main.TRUE, actual=hostResult, 582 onpass="Found a host id for each host", 583 onfail="Error looking up host ids" ) 584 585 intentStart = time.time() 586 onosIds = main.ONOScli1.getAllIntentsId() 587 main.log.info( "Submitted intents: " + str( intentIds ) ) 588 main.log.info( "Intents in ONOS: " + str( onosIds ) ) 589 for intent in intentIds: 590 if intent in onosIds: 591 pass # intent submitted is in onos 592 else: 593 intentAddResult = False 594 if intentAddResult: 595 intentStop = time.time() 596 else: 597 intentStop = None 598 # Print the intent states 599 intents = main.ONOScli1.intents() 600 intentStates = [] 601 installedCheck = True 602 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 603 count = 0 604 try: 605 for intent in json.loads( intents ): 606 state = intent.get( 'state', None ) 607 if "INSTALLED" not in state: 608 installedCheck = False 609 intentId = intent.get( 'id', None ) 610 intentStates.append( ( intentId, state ) ) 611 except ( ValueError, TypeError ): 612 main.log.exception( "Error parsing intents" ) 613 # add submitted intents not in the store 614 tmplist = [ i for i, s in intentStates ] 615 missingIntents = False 616 for i in intentIds: 617 if i not in tmplist: 618 intentStates.append( ( i, " - " ) ) 619 missingIntents = True 620 intentStates.sort() 621 for i, s in intentStates: 622 count += 1 623 main.log.info( "%-6s%-15s%-15s" % 624 ( str( count ), str( i ), str( s ) ) ) 625 leaders = main.ONOScli1.leaders() 626 try: 627 missing = False 628 if leaders: 629 parsedLeaders = json.loads( leaders ) 630 main.log.warn( json.dumps( parsedLeaders, 631 sort_keys=True, 632 indent=4, 633 separators=( ',', ': ' ) ) ) 634 # check for all intent partitions 635 topics = [] 636 for i in range( 14 ): 637 topics.append( "intent-partition-" + str( i ) ) 638 main.log.debug( topics ) 639 ONOStopics = [ j['topic'] for j in parsedLeaders ] 640 for topic in topics: 641 if topic not in ONOStopics: 642 main.log.error( "Error: " + topic + 643 " not in leaders" ) 644 missing = True 645 else: 646 main.log.error( "leaders() returned None" ) 647 except ( ValueError, TypeError ): 648 main.log.exception( "Error parsing leaders" ) 649 main.log.error( repr( leaders ) ) 650 # Check all nodes 651 if missing: 652 for node in main.CLIs: 653 response = node.leaders( jsonFormat=False) 654 main.log.warn( str( node.name ) + " leaders output: \n" + 655 str( response ) ) 656 657 partitions = main.ONOScli1.partitions() 658 try: 659 if partitions : 660 parsedPartitions = json.loads( partitions ) 661 main.log.warn( json.dumps( parsedPartitions, 662 sort_keys=True, 663 indent=4, 664 separators=( ',', ': ' ) ) ) 665 # TODO check for a leader in all paritions 666 # TODO check for consistency among nodes 667 else: 668 main.log.error( "partitions() returned None" ) 669 except ( ValueError, TypeError ): 670 main.log.exception( "Error parsing partitions" ) 671 main.log.error( repr( partitions ) ) 672 pendingMap = main.ONOScli1.pendingMap() 673 try: 674 if pendingMap : 675 parsedPending = json.loads( pendingMap ) 676 main.log.warn( json.dumps( parsedPending, 677 sort_keys=True, 678 indent=4, 679 separators=( ',', ': ' ) ) ) 680 # TODO check something here? 681 else: 682 main.log.error( "pendingMap() returned None" ) 683 except ( ValueError, TypeError ): 684 main.log.exception( "Error parsing pending map" ) 685 main.log.error( repr( pendingMap ) ) 686 687 intentAddResult = bool( intentAddResult and not missingIntents and 688 installedCheck ) 689 if not intentAddResult: 690 main.log.error( "Error in pushing host intents to ONOS" ) 691 692 main.step( "Intent Anti-Entropy dispersion" ) 693 for i in range(100): 694 correct = True 695 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) ) 696 for cli in main.CLIs: 697 onosIds = [] 698 ids = cli.getAllIntentsId() 699 onosIds.append( ids ) 700 main.log.debug( "Intents in " + cli.name + ": " + 701 str( sorted( onosIds ) ) ) 702 if sorted( ids ) != sorted( intentIds ): 703 main.log.warn( "Set of intent IDs doesn't match" ) 704 correct = False 705 break 706 else: 707 intents = json.loads( cli.intents() ) 708 for intent in intents: 709 if intent[ 'state' ] != "INSTALLED": 710 main.log.warn( "Intent " + intent[ 'id' ] + 711 " is " + intent[ 'state' ] ) 712 correct = False 713 break 714 if correct: 715 break 716 else: 717 time.sleep(1) 718 if not intentStop: 719 intentStop = time.time() 720 global gossipTime 721 gossipTime = intentStop - intentStart 722 main.log.info( "It took about " + str( gossipTime ) + 723 " seconds for all intents to appear in each node" ) 724 # FIXME: make this time configurable/calculate based off of number of 725 # nodes and gossip rounds 726 utilities.assert_greater_equals( 727 expect=40, actual=gossipTime, 728 onpass="ECM anti-entropy for intents worked within " + 729 "expected time", 730 onfail="Intent ECM anti-entropy took too long" ) 731 if gossipTime <= 40: 732 intentAddResult = True 733 734 if not intentAddResult or "key" in pendingMap: 735 import time 736 installedCheck = True 737 main.log.info( "Sleeping 60 seconds to see if intents are found" ) 738 time.sleep( 60 ) 739 onosIds = main.ONOScli1.getAllIntentsId() 740 main.log.info( "Submitted intents: " + str( intentIds ) ) 741 main.log.info( "Intents in ONOS: " + str( onosIds ) ) 742 # Print the intent states 743 intents = main.ONOScli1.intents() 744 intentStates = [] 745 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 746 count = 0 747 try: 748 for intent in json.loads( intents ): 749 # Iter through intents of a node 750 state = intent.get( 'state', None ) 751 if "INSTALLED" not in state: 752 installedCheck = False 753 intentId = intent.get( 'id', None ) 754 intentStates.append( ( intentId, state ) ) 755 except ( ValueError, TypeError ): 756 main.log.exception( "Error parsing intents" ) 757 # add submitted intents not in the store 758 tmplist = [ i for i, s in intentStates ] 759 for i in intentIds: 760 if i not in tmplist: 761 intentStates.append( ( i, " - " ) ) 762 intentStates.sort() 763 for i, s in intentStates: 764 count += 1 765 main.log.info( "%-6s%-15s%-15s" % 766 ( str( count ), str( i ), str( s ) ) ) 767 leaders = main.ONOScli1.leaders() 768 try: 769 missing = False 770 if leaders: 771 parsedLeaders = json.loads( leaders ) 772 main.log.warn( json.dumps( parsedLeaders, 773 sort_keys=True, 774 indent=4, 775 separators=( ',', ': ' ) ) ) 776 # check for all intent partitions 777 # check for election 778 topics = [] 779 for i in range( 14 ): 780 topics.append( "intent-partition-" + str( i ) ) 781 # FIXME: this should only be after we start the app 782 topics.append( "org.onosproject.election" ) 783 main.log.debug( topics ) 784 ONOStopics = [ j['topic'] for j in parsedLeaders ] 785 for topic in topics: 786 if topic not in ONOStopics: 787 main.log.error( "Error: " + topic + 788 " not in leaders" ) 789 missing = True 790 else: 791 main.log.error( "leaders() returned None" ) 792 except ( ValueError, TypeError ): 793 main.log.exception( "Error parsing leaders" ) 794 main.log.error( repr( leaders ) ) 795 # Check all nodes 796 if missing: 797 for node in main.CLIs: 798 response = node.leaders( jsonFormat=False) 799 main.log.warn( str( node.name ) + " leaders output: \n" + 800 str( response ) ) 801 802 partitions = main.ONOScli1.partitions() 803 try: 804 if partitions : 805 parsedPartitions = json.loads( partitions ) 806 main.log.warn( json.dumps( parsedPartitions, 807 sort_keys=True, 808 indent=4, 809 separators=( ',', ': ' ) ) ) 810 # TODO check for a leader in all paritions 811 # TODO check for consistency among nodes 812 else: 813 main.log.error( "partitions() returned None" ) 814 except ( ValueError, TypeError ): 815 main.log.exception( "Error parsing partitions" ) 816 main.log.error( repr( partitions ) ) 817 pendingMap = main.ONOScli1.pendingMap() 818 try: 819 if pendingMap : 820 parsedPending = json.loads( pendingMap ) 821 main.log.warn( json.dumps( parsedPending, 822 sort_keys=True, 823 indent=4, 824 separators=( ',', ': ' ) ) ) 825 # TODO check something here? 826 else: 827 main.log.error( "pendingMap() returned None" ) 828 except ( ValueError, TypeError ): 829 main.log.exception( "Error parsing pending map" ) 830 main.log.error( repr( pendingMap ) )
831
832 - def CASE4( self, main ):
833 """ 834 Ping across added host intents 835 """ 836 import json 837 import time 838 assert main.numCtrls, "main.numCtrls not defined" 839 assert main, "main not defined" 840 assert utilities.assert_equals, "utilities.assert_equals not defined" 841 assert main.CLIs, "main.CLIs not defined" 842 assert main.nodes, "main.nodes not defined" 843 main.case( "Verify connectivity by sendind traffic across Intents" ) 844 main.caseExplanation = "Ping across added host intents to check " +\ 845 "functionality and check the state of " +\ 846 "the intent" 847 main.step( "Ping across added host intents" ) 848 PingResult = main.TRUE 849 for i in range( 8, 18 ): 850 ping = main.Mininet1.pingHost( src="h" + str( i ), 851 target="h" + str( i + 10 ) ) 852 PingResult = PingResult and ping 853 if ping == main.FALSE: 854 main.log.warn( "Ping failed between h" + str( i ) + 855 " and h" + str( i + 10 ) ) 856 elif ping == main.TRUE: 857 main.log.info( "Ping test passed!" ) 858 # Don't set PingResult or you'd override failures 859 if PingResult == main.FALSE: 860 main.log.error( 861 "Intents have not been installed correctly, pings failed." ) 862 # TODO: pretty print 863 main.log.warn( "ONOS1 intents: " ) 864 try: 865 tmpIntents = main.ONOScli1.intents() 866 main.log.warn( json.dumps( json.loads( tmpIntents ), 867 sort_keys=True, 868 indent=4, 869 separators=( ',', ': ' ) ) ) 870 except ( ValueError, TypeError ): 871 main.log.warn( repr( tmpIntents ) ) 872 utilities.assert_equals( 873 expect=main.TRUE, 874 actual=PingResult, 875 onpass="Intents have been installed correctly and pings work", 876 onfail="Intents have not been installed correctly, pings failed." ) 877 878 main.step( "Check Intent state" ) 879 installedCheck = False 880 loopCount = 0 881 while not installedCheck and loopCount < 40: 882 installedCheck = True 883 # Print the intent states 884 intents = main.ONOScli1.intents() 885 intentStates = [] 886 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 887 count = 0 888 # Iter through intents of a node 889 try: 890 for intent in json.loads( intents ): 891 state = intent.get( 'state', None ) 892 if "INSTALLED" not in state: 893 installedCheck = False 894 intentId = intent.get( 'id', None ) 895 intentStates.append( ( intentId, state ) ) 896 except ( ValueError, TypeError ): 897 main.log.exception( "Error parsing intents." ) 898 # Print states 899 intentStates.sort() 900 for i, s in intentStates: 901 count += 1 902 main.log.info( "%-6s%-15s%-15s" % 903 ( str( count ), str( i ), str( s ) ) ) 904 if not installedCheck: 905 time.sleep( 1 ) 906 loopCount += 1 907 utilities.assert_equals( expect=True, actual=installedCheck, 908 onpass="Intents are all INSTALLED", 909 onfail="Intents are not all in " + 910 "INSTALLED state" ) 911 912 main.step( "Check leadership of topics" ) 913 leaders = main.ONOScli1.leaders() 914 topicCheck = main.TRUE 915 try: 916 if leaders: 917 parsedLeaders = json.loads( leaders ) 918 main.log.warn( json.dumps( parsedLeaders, 919 sort_keys=True, 920 indent=4, 921 separators=( ',', ': ' ) ) ) 922 # check for all intent partitions 923 # check for election 924 # TODO: Look at Devices as topics now that it uses this system 925 topics = [] 926 for i in range( 14 ): 927 topics.append( "intent-partition-" + str( i ) ) 928 # FIXME: this should only be after we start the app 929 # FIXME: topics.append( "org.onosproject.election" ) 930 # Print leaders output 931 main.log.debug( topics ) 932 ONOStopics = [ j['topic'] for j in parsedLeaders ] 933 for topic in topics: 934 if topic not in ONOStopics: 935 main.log.error( "Error: " + topic + 936 " not in leaders" ) 937 topicCheck = main.FALSE 938 else: 939 main.log.error( "leaders() returned None" ) 940 topicCheck = main.FALSE 941 except ( ValueError, TypeError ): 942 topicCheck = main.FALSE 943 main.log.exception( "Error parsing leaders" ) 944 main.log.error( repr( leaders ) ) 945 # TODO: Check for a leader of these topics 946 # Check all nodes 947 if topicCheck: 948 for node in main.CLIs: 949 response = node.leaders( jsonFormat=False) 950 main.log.warn( str( node.name ) + " leaders output: \n" + 951 str( response ) ) 952 953 utilities.assert_equals( expect=main.TRUE, actual=topicCheck, 954 onpass="intent Partitions is in leaders", 955 onfail="Some topics were lost " ) 956 # Print partitions 957 partitions = main.ONOScli1.partitions() 958 try: 959 if partitions : 960 parsedPartitions = json.loads( partitions ) 961 main.log.warn( json.dumps( parsedPartitions, 962 sort_keys=True, 963 indent=4, 964 separators=( ',', ': ' ) ) ) 965 # TODO check for a leader in all paritions 966 # TODO check for consistency among nodes 967 else: 968 main.log.error( "partitions() returned None" ) 969 except ( ValueError, TypeError ): 970 main.log.exception( "Error parsing partitions" ) 971 main.log.error( repr( partitions ) ) 972 # Print Pending Map 973 pendingMap = main.ONOScli1.pendingMap() 974 try: 975 if pendingMap : 976 parsedPending = json.loads( pendingMap ) 977 main.log.warn( json.dumps( parsedPending, 978 sort_keys=True, 979 indent=4, 980 separators=( ',', ': ' ) ) ) 981 # TODO check something here? 982 else: 983 main.log.error( "pendingMap() returned None" ) 984 except ( ValueError, TypeError ): 985 main.log.exception( "Error parsing pending map" ) 986 main.log.error( repr( pendingMap ) ) 987 988 if not installedCheck: 989 main.log.info( "Waiting 60 seconds to see if the state of " + 990 "intents change" ) 991 time.sleep( 60 ) 992 # Print the intent states 993 intents = main.ONOScli1.intents() 994 intentStates = [] 995 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 996 count = 0 997 # Iter through intents of a node 998 try: 999 for intent in json.loads( intents ): 1000 state = intent.get( 'state', None ) 1001 if "INSTALLED" not in state: 1002 installedCheck = False 1003 intentId = intent.get( 'id', None ) 1004 intentStates.append( ( intentId, state ) ) 1005 except ( ValueError, TypeError ): 1006 main.log.exception( "Error parsing intents." ) 1007 intentStates.sort() 1008 for i, s in intentStates: 1009 count += 1 1010 main.log.info( "%-6s%-15s%-15s" % 1011 ( str( count ), str( i ), str( s ) ) ) 1012 leaders = main.ONOScli1.leaders() 1013 try: 1014 missing = False 1015 if leaders: 1016 parsedLeaders = json.loads( leaders ) 1017 main.log.warn( json.dumps( parsedLeaders, 1018 sort_keys=True, 1019 indent=4, 1020 separators=( ',', ': ' ) ) ) 1021 # check for all intent partitions 1022 # check for election 1023 topics = [] 1024 for i in range( 14 ): 1025 topics.append( "intent-partition-" + str( i ) ) 1026 # FIXME: this should only be after we start the app 1027 topics.append( "org.onosproject.election" ) 1028 main.log.debug( topics ) 1029 ONOStopics = [ j['topic'] for j in parsedLeaders ] 1030 for topic in topics: 1031 if topic not in ONOStopics: 1032 main.log.error( "Error: " + topic + 1033 " not in leaders" ) 1034 missing = True 1035 else: 1036 main.log.error( "leaders() returned None" ) 1037 except ( ValueError, TypeError ): 1038 main.log.exception( "Error parsing leaders" ) 1039 main.log.error( repr( leaders ) ) 1040 if missing: 1041 for node in main.CLIs: 1042 response = node.leaders( jsonFormat=False) 1043 main.log.warn( str( node.name ) + " leaders output: \n" + 1044 str( response ) ) 1045 1046 partitions = main.ONOScli1.partitions() 1047 try: 1048 if partitions : 1049 parsedPartitions = json.loads( partitions ) 1050 main.log.warn( json.dumps( parsedPartitions, 1051 sort_keys=True, 1052 indent=4, 1053 separators=( ',', ': ' ) ) ) 1054 # TODO check for a leader in all paritions 1055 # TODO check for consistency among nodes 1056 else: 1057 main.log.error( "partitions() returned None" ) 1058 except ( ValueError, TypeError ): 1059 main.log.exception( "Error parsing partitions" ) 1060 main.log.error( repr( partitions ) ) 1061 pendingMap = main.ONOScli1.pendingMap() 1062 try: 1063 if pendingMap : 1064 parsedPending = json.loads( pendingMap ) 1065 main.log.warn( json.dumps( parsedPending, 1066 sort_keys=True, 1067 indent=4, 1068 separators=( ',', ': ' ) ) ) 1069 # TODO check something here? 1070 else: 1071 main.log.error( "pendingMap() returned None" ) 1072 except ( ValueError, TypeError ): 1073 main.log.exception( "Error parsing pending map" ) 1074 main.log.error( repr( pendingMap ) ) 1075 # Print flowrules 1076 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) ) 1077 main.step( "Wait a minute then ping again" ) 1078 # the wait is above 1079 PingResult = main.TRUE 1080 for i in range( 8, 18 ): 1081 ping = main.Mininet1.pingHost( src="h" + str( i ), 1082 target="h" + str( i + 10 ) ) 1083 PingResult = PingResult and ping 1084 if ping == main.FALSE: 1085 main.log.warn( "Ping failed between h" + str( i ) + 1086 " and h" + str( i + 10 ) ) 1087 elif ping == main.TRUE: 1088 main.log.info( "Ping test passed!" ) 1089 # Don't set PingResult or you'd override failures 1090 if PingResult == main.FALSE: 1091 main.log.error( 1092 "Intents have not been installed correctly, pings failed." ) 1093 # TODO: pretty print 1094 main.log.warn( "ONOS1 intents: " ) 1095 try: 1096 tmpIntents = main.ONOScli1.intents() 1097 main.log.warn( json.dumps( json.loads( tmpIntents ), 1098 sort_keys=True, 1099 indent=4, 1100 separators=( ',', ': ' ) ) ) 1101 except ( ValueError, TypeError ): 1102 main.log.warn( repr( tmpIntents ) ) 1103 utilities.assert_equals( 1104 expect=main.TRUE, 1105 actual=PingResult, 1106 onpass="Intents have been installed correctly and pings work", 1107 onfail="Intents have not been installed correctly, pings failed." )
1108
1109 - def CASE5( self, main ):
1110 """ 1111 Reading state of ONOS 1112 """ 1113 import json 1114 import time 1115 assert main.numCtrls, "main.numCtrls not defined" 1116 assert main, "main not defined" 1117 assert utilities.assert_equals, "utilities.assert_equals not defined" 1118 assert main.CLIs, "main.CLIs not defined" 1119 assert main.nodes, "main.nodes not defined" 1120 1121 main.case( "Setting up and gathering data for current state" ) 1122 # The general idea for this test case is to pull the state of 1123 # ( intents,flows, topology,... ) from each ONOS node 1124 # We can then compare them with each other and also with past states 1125 1126 main.step( "Check that each switch has a master" ) 1127 global mastershipState 1128 mastershipState = '[]' 1129 1130 # Assert that each device has a master 1131 rolesNotNull = main.TRUE 1132 threads = [] 1133 for i in range( main.numCtrls ): 1134 t = main.Thread( target=main.CLIs[i].rolesNotNull, 1135 name="rolesNotNull-" + str( i ), 1136 args=[] ) 1137 threads.append( t ) 1138 t.start() 1139 1140 for t in threads: 1141 t.join() 1142 rolesNotNull = rolesNotNull and t.result 1143 utilities.assert_equals( 1144 expect=main.TRUE, 1145 actual=rolesNotNull, 1146 onpass="Each device has a master", 1147 onfail="Some devices don't have a master assigned" ) 1148 1149 main.step( "Get the Mastership of each switch from each controller" ) 1150 ONOSMastership = [] 1151 mastershipCheck = main.FALSE 1152 consistentMastership = True 1153 rolesResults = True 1154 threads = [] 1155 for i in range( main.numCtrls ): 1156 t = main.Thread( target=main.CLIs[i].roles, 1157 name="roles-" + str( i ), 1158 args=[] ) 1159 threads.append( t ) 1160 t.start() 1161 1162 for t in threads: 1163 t.join() 1164 ONOSMastership.append( t.result ) 1165 1166 for i in range( main.numCtrls ): 1167 if not ONOSMastership[i] or "Error" in ONOSMastership[i]: 1168 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1169 " roles" ) 1170 main.log.warn( 1171 "ONOS" + str( i + 1 ) + " mastership response: " + 1172 repr( ONOSMastership[i] ) ) 1173 rolesResults = False 1174 utilities.assert_equals( 1175 expect=True, 1176 actual=rolesResults, 1177 onpass="No error in reading roles output", 1178 onfail="Error in reading roles from ONOS" ) 1179 1180 main.step( "Check for consistency in roles from each controller" ) 1181 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ): 1182 main.log.info( 1183 "Switch roles are consistent across all ONOS nodes" ) 1184 else: 1185 consistentMastership = False 1186 utilities.assert_equals( 1187 expect=True, 1188 actual=consistentMastership, 1189 onpass="Switch roles are consistent across all ONOS nodes", 1190 onfail="ONOS nodes have different views of switch roles" ) 1191 1192 if rolesResults and not consistentMastership: 1193 for i in range( main.numCtrls ): 1194 try: 1195 main.log.warn( 1196 "ONOS" + str( i + 1 ) + " roles: ", 1197 json.dumps( 1198 json.loads( ONOSMastership[ i ] ), 1199 sort_keys=True, 1200 indent=4, 1201 separators=( ',', ': ' ) ) ) 1202 except ( ValueError, TypeError ): 1203 main.log.warn( repr( ONOSMastership[ i ] ) ) 1204 elif rolesResults and consistentMastership: 1205 mastershipCheck = main.TRUE 1206 mastershipState = ONOSMastership[ 0 ] 1207 1208 main.step( "Get the intents from each controller" ) 1209 global intentState 1210 intentState = [] 1211 ONOSIntents = [] 1212 intentCheck = main.FALSE 1213 consistentIntents = True 1214 intentsResults = True 1215 threads = [] 1216 for i in range( main.numCtrls ): 1217 t = main.Thread( target=main.CLIs[i].intents, 1218 name="intents-" + str( i ), 1219 args=[], 1220 kwargs={ 'jsonFormat': True } ) 1221 threads.append( t ) 1222 t.start() 1223 1224 for t in threads: 1225 t.join() 1226 ONOSIntents.append( t.result ) 1227 1228 for i in range( main.numCtrls ): 1229 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]: 1230 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1231 " intents" ) 1232 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " + 1233 repr( ONOSIntents[ i ] ) ) 1234 intentsResults = False 1235 utilities.assert_equals( 1236 expect=True, 1237 actual=intentsResults, 1238 onpass="No error in reading intents output", 1239 onfail="Error in reading intents from ONOS" ) 1240 1241 main.step( "Check for consistency in Intents from each controller" ) 1242 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ): 1243 main.log.info( "Intents are consistent across all ONOS " + 1244 "nodes" ) 1245 else: 1246 consistentIntents = False 1247 main.log.error( "Intents not consistent" ) 1248 utilities.assert_equals( 1249 expect=True, 1250 actual=consistentIntents, 1251 onpass="Intents are consistent across all ONOS nodes", 1252 onfail="ONOS nodes have different views of intents" ) 1253 1254 if intentsResults: 1255 # Try to make it easy to figure out what is happening 1256 # 1257 # Intent ONOS1 ONOS2 ... 1258 # 0x01 INSTALLED INSTALLING 1259 # ... ... ... 1260 # ... ... ... 1261 title = " Id" 1262 for n in range( main.numCtrls ): 1263 title += " " * 10 + "ONOS" + str( n + 1 ) 1264 main.log.warn( title ) 1265 # get all intent keys in the cluster 1266 keys = [] 1267 for nodeStr in ONOSIntents: 1268 node = json.loads( nodeStr ) 1269 for intent in node: 1270 keys.append( intent.get( 'id' ) ) 1271 keys = set( keys ) 1272 for key in keys: 1273 row = "%-13s" % key 1274 for nodeStr in ONOSIntents: 1275 node = json.loads( nodeStr ) 1276 for intent in node: 1277 if intent.get( 'id', "Error" ) == key: 1278 row += "%-15s" % intent.get( 'state' ) 1279 main.log.warn( row ) 1280 # End table view 1281 1282 if intentsResults and not consistentIntents: 1283 # print the json objects 1284 n = len(ONOSIntents) 1285 main.log.debug( "ONOS" + str( n ) + " intents: " ) 1286 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ), 1287 sort_keys=True, 1288 indent=4, 1289 separators=( ',', ': ' ) ) ) 1290 for i in range( main.numCtrls ): 1291 if ONOSIntents[ i ] != ONOSIntents[ -1 ]: 1292 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " ) 1293 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ), 1294 sort_keys=True, 1295 indent=4, 1296 separators=( ',', ': ' ) ) ) 1297 else: 1298 main.log.debug( main.nodes[ i ].name + " intents match ONOS" + 1299 str( n ) + " intents" ) 1300 elif intentsResults and consistentIntents: 1301 intentCheck = main.TRUE 1302 intentState = ONOSIntents[ 0 ] 1303 1304 main.step( "Get the flows from each controller" ) 1305 global flowState 1306 flowState = [] 1307 ONOSFlows = [] 1308 ONOSFlowsJson = [] 1309 flowCheck = main.FALSE 1310 consistentFlows = True 1311 flowsResults = True 1312 threads = [] 1313 for i in range( main.numCtrls ): 1314 t = main.Thread( target=main.CLIs[i].flows, 1315 name="flows-" + str( i ), 1316 args=[], 1317 kwargs={ 'jsonFormat': True } ) 1318 threads.append( t ) 1319 t.start() 1320 1321 # NOTE: Flows command can take some time to run 1322 time.sleep(30) 1323 for t in threads: 1324 t.join() 1325 result = t.result 1326 ONOSFlows.append( result ) 1327 1328 for i in range( main.numCtrls ): 1329 num = str( i + 1 ) 1330 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]: 1331 main.log.error( "Error in getting ONOS" + num + " flows" ) 1332 main.log.warn( "ONOS" + num + " flows response: " + 1333 repr( ONOSFlows[ i ] ) ) 1334 flowsResults = False 1335 ONOSFlowsJson.append( None ) 1336 else: 1337 try: 1338 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) ) 1339 except ( ValueError, TypeError ): 1340 # FIXME: change this to log.error? 1341 main.log.exception( "Error in parsing ONOS" + num + 1342 " response as json." ) 1343 main.log.error( repr( ONOSFlows[ i ] ) ) 1344 ONOSFlowsJson.append( None ) 1345 flowsResults = False 1346 utilities.assert_equals( 1347 expect=True, 1348 actual=flowsResults, 1349 onpass="No error in reading flows output", 1350 onfail="Error in reading flows from ONOS" ) 1351 1352 main.step( "Check for consistency in Flows from each controller" ) 1353 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ] 1354 if all( tmp ): 1355 main.log.info( "Flow count is consistent across all ONOS nodes" ) 1356 else: 1357 consistentFlows = False 1358 utilities.assert_equals( 1359 expect=True, 1360 actual=consistentFlows, 1361 onpass="The flow count is consistent across all ONOS nodes", 1362 onfail="ONOS nodes have different flow counts" ) 1363 1364 if flowsResults and not consistentFlows: 1365 for i in range( main.numCtrls ): 1366 try: 1367 main.log.warn( 1368 "ONOS" + str( i + 1 ) + " flows: " + 1369 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True, 1370 indent=4, separators=( ',', ': ' ) ) ) 1371 except ( ValueError, TypeError ): 1372 main.log.warn( 1373 "ONOS" + str( i + 1 ) + " flows: " + 1374 repr( ONOSFlows[ i ] ) ) 1375 elif flowsResults and consistentFlows: 1376 flowCheck = main.TRUE 1377 flowState = ONOSFlows[ 0 ] 1378 1379 main.step( "Get the OF Table entries" ) 1380 global flows 1381 flows = [] 1382 for i in range( 1, 29 ): 1383 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) ) 1384 if flowCheck == main.FALSE: 1385 for table in flows: 1386 main.log.warn( table ) 1387 # TODO: Compare switch flow tables with ONOS flow tables 1388 1389 main.step( "Start continuous pings" ) 1390 main.Mininet2.pingLong( 1391 src=main.params[ 'PING' ][ 'source1' ], 1392 target=main.params[ 'PING' ][ 'target1' ], 1393 pingTime=500 ) 1394 main.Mininet2.pingLong( 1395 src=main.params[ 'PING' ][ 'source2' ], 1396 target=main.params[ 'PING' ][ 'target2' ], 1397 pingTime=500 ) 1398 main.Mininet2.pingLong( 1399 src=main.params[ 'PING' ][ 'source3' ], 1400 target=main.params[ 'PING' ][ 'target3' ], 1401 pingTime=500 ) 1402 main.Mininet2.pingLong( 1403 src=main.params[ 'PING' ][ 'source4' ], 1404 target=main.params[ 'PING' ][ 'target4' ], 1405 pingTime=500 ) 1406 main.Mininet2.pingLong( 1407 src=main.params[ 'PING' ][ 'source5' ], 1408 target=main.params[ 'PING' ][ 'target5' ], 1409 pingTime=500 ) 1410 main.Mininet2.pingLong( 1411 src=main.params[ 'PING' ][ 'source6' ], 1412 target=main.params[ 'PING' ][ 'target6' ], 1413 pingTime=500 ) 1414 main.Mininet2.pingLong( 1415 src=main.params[ 'PING' ][ 'source7' ], 1416 target=main.params[ 'PING' ][ 'target7' ], 1417 pingTime=500 ) 1418 main.Mininet2.pingLong( 1419 src=main.params[ 'PING' ][ 'source8' ], 1420 target=main.params[ 'PING' ][ 'target8' ], 1421 pingTime=500 ) 1422 main.Mininet2.pingLong( 1423 src=main.params[ 'PING' ][ 'source9' ], 1424 target=main.params[ 'PING' ][ 'target9' ], 1425 pingTime=500 ) 1426 main.Mininet2.pingLong( 1427 src=main.params[ 'PING' ][ 'source10' ], 1428 target=main.params[ 'PING' ][ 'target10' ], 1429 pingTime=500 ) 1430 1431 main.step( "Collecting topology information from ONOS" ) 1432 devices = [] 1433 threads = [] 1434 for i in range( main.numCtrls ): 1435 t = main.Thread( target=main.CLIs[i].devices, 1436 name="devices-" + str( i ), 1437 args=[ ] ) 1438 threads.append( t ) 1439 t.start() 1440 1441 for t in threads: 1442 t.join() 1443 devices.append( t.result ) 1444 hosts = [] 1445 threads = [] 1446 for i in range( main.numCtrls ): 1447 t = main.Thread( target=main.CLIs[i].hosts, 1448 name="hosts-" + str( i ), 1449 args=[ ] ) 1450 threads.append( t ) 1451 t.start() 1452 1453 for t in threads: 1454 t.join() 1455 try: 1456 hosts.append( json.loads( t.result ) ) 1457 except ( ValueError, TypeError ): 1458 # FIXME: better handling of this, print which node 1459 # Maybe use thread name? 1460 main.log.exception( "Error parsing json output of hosts" ) 1461 # FIXME: should this be an empty json object instead? 1462 hosts.append( None ) 1463 1464 ports = [] 1465 threads = [] 1466 for i in range( main.numCtrls ): 1467 t = main.Thread( target=main.CLIs[i].ports, 1468 name="ports-" + str( i ), 1469 args=[ ] ) 1470 threads.append( t ) 1471 t.start() 1472 1473 for t in threads: 1474 t.join() 1475 ports.append( t.result ) 1476 links = [] 1477 threads = [] 1478 for i in range( main.numCtrls ): 1479 t = main.Thread( target=main.CLIs[i].links, 1480 name="links-" + str( i ), 1481 args=[ ] ) 1482 threads.append( t ) 1483 t.start() 1484 1485 for t in threads: 1486 t.join() 1487 links.append( t.result ) 1488 clusters = [] 1489 threads = [] 1490 for i in range( main.numCtrls ): 1491 t = main.Thread( target=main.CLIs[i].clusters, 1492 name="clusters-" + str( i ), 1493 args=[ ] ) 1494 threads.append( t ) 1495 t.start() 1496 1497 for t in threads: 1498 t.join() 1499 clusters.append( t.result ) 1500 # Compare json objects for hosts and dataplane clusters 1501 1502 # hosts 1503 main.step( "Host view is consistent across ONOS nodes" ) 1504 consistentHostsResult = main.TRUE 1505 for controller in range( len( hosts ) ): 1506 controllerStr = str( controller + 1 ) 1507 if "Error" not in hosts[ controller ]: 1508 if hosts[ controller ] == hosts[ 0 ]: 1509 continue 1510 else: # hosts not consistent 1511 main.log.error( "hosts from ONOS" + 1512 controllerStr + 1513 " is inconsistent with ONOS1" ) 1514 main.log.warn( repr( hosts[ controller ] ) ) 1515 consistentHostsResult = main.FALSE 1516 1517 else: 1518 main.log.error( "Error in getting ONOS hosts from ONOS" + 1519 controllerStr ) 1520 consistentHostsResult = main.FALSE 1521 main.log.warn( "ONOS" + controllerStr + 1522 " hosts response: " + 1523 repr( hosts[ controller ] ) ) 1524 utilities.assert_equals( 1525 expect=main.TRUE, 1526 actual=consistentHostsResult, 1527 onpass="Hosts view is consistent across all ONOS nodes", 1528 onfail="ONOS nodes have different views of hosts" ) 1529 1530 main.step( "Each host has an IP address" ) 1531 ipResult = main.TRUE 1532 for controller in range( 0, len( hosts ) ): 1533 controllerStr = str( controller + 1 ) 1534 for host in hosts[ controller ]: 1535 if not host.get( 'ipAddresses', [ ] ): 1536 main.log.error( "DEBUG:Error with host ips on controller" + 1537 controllerStr + ": " + str( host ) ) 1538 ipResult = main.FALSE 1539 utilities.assert_equals( 1540 expect=main.TRUE, 1541 actual=ipResult, 1542 onpass="The ips of the hosts aren't empty", 1543 onfail="The ip of at least one host is missing" ) 1544 1545 # Strongly connected clusters of devices 1546 main.step( "Cluster view is consistent across ONOS nodes" ) 1547 consistentClustersResult = main.TRUE 1548 for controller in range( len( clusters ) ): 1549 controllerStr = str( controller + 1 ) 1550 if "Error" not in clusters[ controller ]: 1551 if clusters[ controller ] == clusters[ 0 ]: 1552 continue 1553 else: # clusters not consistent 1554 main.log.error( "clusters from ONOS" + controllerStr + 1555 " is inconsistent with ONOS1" ) 1556 consistentClustersResult = main.FALSE 1557 1558 else: 1559 main.log.error( "Error in getting dataplane clusters " + 1560 "from ONOS" + controllerStr ) 1561 consistentClustersResult = main.FALSE 1562 main.log.warn( "ONOS" + controllerStr + 1563 " clusters response: " + 1564 repr( clusters[ controller ] ) ) 1565 utilities.assert_equals( 1566 expect=main.TRUE, 1567 actual=consistentClustersResult, 1568 onpass="Clusters view is consistent across all ONOS nodes", 1569 onfail="ONOS nodes have different views of clusters" ) 1570 # there should always only be one cluster 1571 main.step( "Cluster view correct across ONOS nodes" ) 1572 try: 1573 numClusters = len( json.loads( clusters[ 0 ] ) ) 1574 except ( ValueError, TypeError ): 1575 main.log.exception( "Error parsing clusters[0]: " + 1576 repr( clusters[ 0 ] ) ) 1577 clusterResults = main.FALSE 1578 if numClusters == 1: 1579 clusterResults = main.TRUE 1580 utilities.assert_equals( 1581 expect=1, 1582 actual=numClusters, 1583 onpass="ONOS shows 1 SCC", 1584 onfail="ONOS shows " + str( numClusters ) + " SCCs" ) 1585 1586 main.step( "Comparing ONOS topology to MN" ) 1587 devicesResults = main.TRUE 1588 linksResults = main.TRUE 1589 hostsResults = main.TRUE 1590 mnSwitches = main.Mininet1.getSwitches() 1591 mnLinks = main.Mininet1.getLinks() 1592 mnHosts = main.Mininet1.getHosts() 1593 for controller in range( main.numCtrls ): 1594 controllerStr = str( controller + 1 ) 1595 if devices[ controller ] and ports[ controller ] and\ 1596 "Error" not in devices[ controller ] and\ 1597 "Error" not in ports[ controller ]: 1598 1599 currentDevicesResult = main.Mininet1.compareSwitches( 1600 mnSwitches, 1601 json.loads( devices[ controller ] ), 1602 json.loads( ports[ controller ] ) ) 1603 else: 1604 currentDevicesResult = main.FALSE 1605 utilities.assert_equals( expect=main.TRUE, 1606 actual=currentDevicesResult, 1607 onpass="ONOS" + controllerStr + 1608 " Switches view is correct", 1609 onfail="ONOS" + controllerStr + 1610 " Switches view is incorrect" ) 1611 if links[ controller ] and "Error" not in links[ controller ]: 1612 currentLinksResult = main.Mininet1.compareLinks( 1613 mnSwitches, mnLinks, 1614 json.loads( links[ controller ] ) ) 1615 else: 1616 currentLinksResult = main.FALSE 1617 utilities.assert_equals( expect=main.TRUE, 1618 actual=currentLinksResult, 1619 onpass="ONOS" + controllerStr + 1620 " links view is correct", 1621 onfail="ONOS" + controllerStr + 1622 " links view is incorrect" ) 1623 1624 if hosts[ controller ] or "Error" not in hosts[ controller ]: 1625 currentHostsResult = main.Mininet1.compareHosts( 1626 mnHosts, 1627 hosts[ controller ] ) 1628 else: 1629 currentHostsResult = main.FALSE 1630 utilities.assert_equals( expect=main.TRUE, 1631 actual=currentHostsResult, 1632 onpass="ONOS" + controllerStr + 1633 " hosts exist in Mininet", 1634 onfail="ONOS" + controllerStr + 1635 " hosts don't match Mininet" ) 1636 1637 devicesResults = devicesResults and currentDevicesResult 1638 linksResults = linksResults and currentLinksResult 1639 hostsResults = hostsResults and currentHostsResult 1640 1641 main.step( "Device information is correct" ) 1642 utilities.assert_equals( 1643 expect=main.TRUE, 1644 actual=devicesResults, 1645 onpass="Device information is correct", 1646 onfail="Device information is incorrect" ) 1647 1648 main.step( "Links are correct" ) 1649 utilities.assert_equals( 1650 expect=main.TRUE, 1651 actual=linksResults, 1652 onpass="Link are correct", 1653 onfail="Links are incorrect" ) 1654 1655 main.step( "Hosts are correct" ) 1656 utilities.assert_equals( 1657 expect=main.TRUE, 1658 actual=hostsResults, 1659 onpass="Hosts are correct", 1660 onfail="Hosts are incorrect" )
1661
1662 - def CASE6( self, main ):
1663 """ 1664 The Failure case. 1665 """ 1666 import time 1667 assert main.numCtrls, "main.numCtrls not defined" 1668 assert main, "main not defined" 1669 assert utilities.assert_equals, "utilities.assert_equals not defined" 1670 assert main.CLIs, "main.CLIs not defined" 1671 assert main.nodes, "main.nodes not defined" 1672 main.case( "Restart minority of ONOS nodes" ) 1673 main.step( "Killing 3 ONOS nodes" ) 1674 killTime = time.time() 1675 # TODO: Randomize these nodes or base this on partitions 1676 # TODO: use threads in this case 1677 killResults = main.ONOSbench.onosKill( main.nodes[0].ip_address ) 1678 time.sleep( 10 ) 1679 killResults = killResults and\ 1680 main.ONOSbench.onosKill( main.nodes[1].ip_address ) 1681 time.sleep( 10 ) 1682 killResults = killResults and\ 1683 main.ONOSbench.onosKill( main.nodes[2].ip_address ) 1684 utilities.assert_equals( expect=main.TRUE, actual=killResults, 1685 onpass="ONOS Killed successfully", 1686 onfail="ONOS kill NOT successful" ) 1687 1688 main.step( "Checking if ONOS is up yet" ) 1689 count = 0 1690 onosIsupResult = main.FALSE 1691 while onosIsupResult == main.FALSE and count < 10: 1692 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address ) 1693 onos2Isup = main.ONOSbench.isup( main.nodes[1].ip_address ) 1694 onos3Isup = main.ONOSbench.isup( main.nodes[2].ip_address ) 1695 onosIsupResult = onos1Isup and onos2Isup and onos3Isup 1696 count = count + 1 1697 # TODO: if it becomes an issue, we can retry this step a few times 1698 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult, 1699 onpass="ONOS restarted successfully", 1700 onfail="ONOS restart NOT successful" ) 1701 1702 main.step( "Restarting ONOS main.CLIs" ) 1703 cliResult1 = main.ONOScli1.startOnosCli( main.nodes[0].ip_address ) 1704 cliResult2 = main.ONOScli2.startOnosCli( main.nodes[1].ip_address ) 1705 cliResult3 = main.ONOScli3.startOnosCli( main.nodes[2].ip_address ) 1706 cliResults = cliResult1 and cliResult2 and cliResult3 1707 utilities.assert_equals( expect=main.TRUE, actual=cliResults, 1708 onpass="ONOS cli restarted", 1709 onfail="ONOS cli did not restart" ) 1710 1711 # Grab the time of restart so we chan check how long the gossip 1712 # protocol has had time to work 1713 main.restartTime = time.time() - killTime 1714 main.log.debug( "Restart time: " + str( main.restartTime ) ) 1715 ''' 1716 # FIXME: revisit test plan for election with madan 1717 # Rerun for election on restarted nodes 1718 run1 = main.CLIs[0].electionTestRun() 1719 run2 = main.CLIs[1].electionTestRun() 1720 run3 = main.CLIs[2].electionTestRun() 1721 runResults = run1 and run2 and run3 1722 utilities.assert_equals( expect=main.TRUE, actual=runResults, 1723 onpass="Reran for election", 1724 onfail="Failed to rerun for election" ) 1725 ''' 1726 # TODO: MAke this configurable. Also, we are breaking the above timer 1727 time.sleep( 60 ) 1728 main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) ) 1729 main.log.debug( main.CLIs[0].leaders( jsonFormat=False ) ) 1730 main.log.debug( main.CLIs[0].partitions( jsonFormat=False ) )
1731
1732 - def CASE7( self, main ):
1733 """ 1734 Check state after ONOS failure 1735 """ 1736 import json 1737 assert main.numCtrls, "main.numCtrls not defined" 1738 assert main, "main not defined" 1739 assert utilities.assert_equals, "utilities.assert_equals not defined" 1740 assert main.CLIs, "main.CLIs not defined" 1741 assert main.nodes, "main.nodes not defined" 1742 main.case( "Running ONOS Constant State Tests" ) 1743 1744 main.step( "Check that each switch has a master" ) 1745 # Assert that each device has a master 1746 rolesNotNull = main.TRUE 1747 threads = [] 1748 for i in range( main.numCtrls ): 1749 t = main.Thread( target=main.CLIs[i].rolesNotNull, 1750 name="rolesNotNull-" + str( i ), 1751 args=[ ] ) 1752 threads.append( t ) 1753 t.start() 1754 1755 for t in threads: 1756 t.join() 1757 rolesNotNull = rolesNotNull and t.result 1758 utilities.assert_equals( 1759 expect=main.TRUE, 1760 actual=rolesNotNull, 1761 onpass="Each device has a master", 1762 onfail="Some devices don't have a master assigned" ) 1763 1764 main.step( "Read device roles from ONOS" ) 1765 ONOSMastership = [] 1766 consistentMastership = True 1767 rolesResults = True 1768 threads = [] 1769 for i in range( main.numCtrls ): 1770 t = main.Thread( target=main.CLIs[i].roles, 1771 name="roles-" + str( i ), 1772 args=[] ) 1773 threads.append( t ) 1774 t.start() 1775 1776 for t in threads: 1777 t.join() 1778 ONOSMastership.append( t.result ) 1779 1780 for i in range( main.numCtrls ): 1781 if not ONOSMastership[i] or "Error" in ONOSMastership[i]: 1782 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1783 " roles" ) 1784 main.log.warn( 1785 "ONOS" + str( i + 1 ) + " mastership response: " + 1786 repr( ONOSMastership[i] ) ) 1787 rolesResults = False 1788 utilities.assert_equals( 1789 expect=True, 1790 actual=rolesResults, 1791 onpass="No error in reading roles output", 1792 onfail="Error in reading roles from ONOS" ) 1793 1794 main.step( "Check for consistency in roles from each controller" ) 1795 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ): 1796 main.log.info( 1797 "Switch roles are consistent across all ONOS nodes" ) 1798 else: 1799 consistentMastership = False 1800 utilities.assert_equals( 1801 expect=True, 1802 actual=consistentMastership, 1803 onpass="Switch roles are consistent across all ONOS nodes", 1804 onfail="ONOS nodes have different views of switch roles" ) 1805 1806 if rolesResults and not consistentMastership: 1807 for i in range( main.numCtrls ): 1808 main.log.warn( 1809 "ONOS" + str( i + 1 ) + " roles: ", 1810 json.dumps( 1811 json.loads( ONOSMastership[ i ] ), 1812 sort_keys=True, 1813 indent=4, 1814 separators=( ',', ': ' ) ) ) 1815 1816 # NOTE: we expect mastership to change on controller failure 1817 ''' 1818 description2 = "Compare switch roles from before failure" 1819 main.step( description2 ) 1820 try: 1821 currentJson = json.loads( ONOSMastership[0] ) 1822 oldJson = json.loads( mastershipState ) 1823 except ( ValueError, TypeError ): 1824 main.log.exception( "Something is wrong with parsing " + 1825 "ONOSMastership[0] or mastershipState" ) 1826 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) ) 1827 main.log.error( "mastershipState" + repr( mastershipState ) ) 1828 main.cleanup() 1829 main.exit() 1830 mastershipCheck = main.TRUE 1831 for i in range( 1, 29 ): 1832 switchDPID = str( 1833 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) ) 1834 current = [ switch[ 'master' ] for switch in currentJson 1835 if switchDPID in switch[ 'id' ] ] 1836 old = [ switch[ 'master' ] for switch in oldJson 1837 if switchDPID in switch[ 'id' ] ] 1838 if current == old: 1839 mastershipCheck = mastershipCheck and main.TRUE 1840 else: 1841 main.log.warn( "Mastership of switch %s changed" % switchDPID ) 1842 mastershipCheck = main.FALSE 1843 utilities.assert_equals( 1844 expect=main.TRUE, 1845 actual=mastershipCheck, 1846 onpass="Mastership of Switches was not changed", 1847 onfail="Mastership of some switches changed" ) 1848 ''' 1849 1850 main.step( "Get the intents and compare across all nodes" ) 1851 ONOSIntents = [] 1852 intentCheck = main.FALSE 1853 consistentIntents = True 1854 intentsResults = True 1855 threads = [] 1856 for i in range( main.numCtrls ): 1857 t = main.Thread( target=main.CLIs[i].intents, 1858 name="intents-" + str( i ), 1859 args=[], 1860 kwargs={ 'jsonFormat': True } ) 1861 threads.append( t ) 1862 t.start() 1863 1864 for t in threads: 1865 t.join() 1866 ONOSIntents.append( t.result ) 1867 1868 for i in range( main.numCtrls ): 1869 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]: 1870 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1871 " intents" ) 1872 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " + 1873 repr( ONOSIntents[ i ] ) ) 1874 intentsResults = False 1875 utilities.assert_equals( 1876 expect=True, 1877 actual=intentsResults, 1878 onpass="No error in reading intents output", 1879 onfail="Error in reading intents from ONOS" ) 1880 1881 main.step( "Check for consistency in Intents from each controller" ) 1882 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ): 1883 main.log.info( "Intents are consistent across all ONOS " + 1884 "nodes" ) 1885 else: 1886 consistentIntents = False 1887 1888 # Try to make it easy to figure out what is happening 1889 # 1890 # Intent ONOS1 ONOS2 ... 1891 # 0x01 INSTALLED INSTALLING 1892 # ... ... ... 1893 # ... ... ... 1894 title = " ID" 1895 for n in range( main.numCtrls ): 1896 title += " " * 10 + "ONOS" + str( n + 1 ) 1897 main.log.warn( title ) 1898 # get all intent keys in the cluster 1899 keys = [] 1900 for nodeStr in ONOSIntents: 1901 node = json.loads( nodeStr ) 1902 for intent in node: 1903 keys.append( intent.get( 'id' ) ) 1904 keys = set( keys ) 1905 for key in keys: 1906 row = "%-13s" % key 1907 for nodeStr in ONOSIntents: 1908 node = json.loads( nodeStr ) 1909 for intent in node: 1910 if intent.get( 'id' ) == key: 1911 row += "%-15s" % intent.get( 'state' ) 1912 main.log.warn( row ) 1913 # End table view 1914 1915 utilities.assert_equals( 1916 expect=True, 1917 actual=consistentIntents, 1918 onpass="Intents are consistent across all ONOS nodes", 1919 onfail="ONOS nodes have different views of intents" ) 1920 intentStates = [] 1921 for node in ONOSIntents: # Iter through ONOS nodes 1922 nodeStates = [] 1923 # Iter through intents of a node 1924 try: 1925 for intent in json.loads( node ): 1926 nodeStates.append( intent[ 'state' ] ) 1927 except ( ValueError, TypeError ): 1928 main.log.exception( "Error in parsing intents" ) 1929 main.log.error( repr( node ) ) 1930 intentStates.append( nodeStates ) 1931 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ] 1932 main.log.info( dict( out ) ) 1933 1934 if intentsResults and not consistentIntents: 1935 for i in range( main.numCtrls ): 1936 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " ) 1937 main.log.warn( json.dumps( 1938 json.loads( ONOSIntents[ i ] ), 1939 sort_keys=True, 1940 indent=4, 1941 separators=( ',', ': ' ) ) ) 1942 elif intentsResults and consistentIntents: 1943 intentCheck = main.TRUE 1944 1945 # NOTE: Store has no durability, so intents are lost across system 1946 # restarts 1947 main.step( "Compare current intents with intents before the failure" ) 1948 # NOTE: this requires case 5 to pass for intentState to be set. 1949 # maybe we should stop the test if that fails? 1950 sameIntents = main.FALSE 1951 if intentState and intentState == ONOSIntents[ 0 ]: 1952 sameIntents = main.TRUE 1953 main.log.info( "Intents are consistent with before failure" ) 1954 # TODO: possibly the states have changed? we may need to figure out 1955 # what the acceptable states are 1956 elif len( intentState ) == len( ONOSIntents[ 0 ] ): 1957 sameIntents = main.TRUE 1958 try: 1959 before = json.loads( intentState ) 1960 after = json.loads( ONOSIntents[ 0 ] ) 1961 for intent in before: 1962 if intent not in after: 1963 sameIntents = main.FALSE 1964 main.log.debug( "Intent is not currently in ONOS " + 1965 "(at least in the same form):" ) 1966 main.log.debug( json.dumps( intent ) ) 1967 except ( ValueError, TypeError ): 1968 main.log.exception( "Exception printing intents" ) 1969 main.log.debug( repr( ONOSIntents[0] ) ) 1970 main.log.debug( repr( intentState ) ) 1971 if sameIntents == main.FALSE: 1972 try: 1973 main.log.debug( "ONOS intents before: " ) 1974 main.log.debug( json.dumps( json.loads( intentState ), 1975 sort_keys=True, indent=4, 1976 separators=( ',', ': ' ) ) ) 1977 main.log.debug( "Current ONOS intents: " ) 1978 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ), 1979 sort_keys=True, indent=4, 1980 separators=( ',', ': ' ) ) ) 1981 except ( ValueError, TypeError ): 1982 main.log.exception( "Exception printing intents" ) 1983 main.log.debug( repr( ONOSIntents[0] ) ) 1984 main.log.debug( repr( intentState ) ) 1985 utilities.assert_equals( 1986 expect=main.TRUE, 1987 actual=sameIntents, 1988 onpass="Intents are consistent with before failure", 1989 onfail="The Intents changed during failure" ) 1990 intentCheck = intentCheck and sameIntents 1991 1992 main.step( "Get the OF Table entries and compare to before " + 1993 "component failure" ) 1994 FlowTables = main.TRUE 1995 flows2 = [] 1996 for i in range( 28 ): 1997 main.log.info( "Checking flow table on s" + str( i + 1 ) ) 1998 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) ) 1999 flows2.append( tmpFlows ) 2000 tempResult = main.Mininet1.flowComp( 2001 flow1=flows[ i ], 2002 flow2=tmpFlows ) 2003 FlowTables = FlowTables and tempResult 2004 if FlowTables == main.FALSE: 2005 main.log.info( "Differences in flow table for switch: s" + 2006 str( i + 1 ) ) 2007 utilities.assert_equals( 2008 expect=main.TRUE, 2009 actual=FlowTables, 2010 onpass="No changes were found in the flow tables", 2011 onfail="Changes were found in the flow tables" ) 2012 2013 main.Mininet2.pingLongKill() 2014 ''' 2015 main.step( "Check the continuous pings to ensure that no packets " + 2016 "were dropped during component failure" ) 2017 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ], 2018 main.params[ 'TESTONIP' ] ) 2019 LossInPings = main.FALSE 2020 # NOTE: checkForLoss returns main.FALSE with 0% packet loss 2021 for i in range( 8, 18 ): 2022 main.log.info( 2023 "Checking for a loss in pings along flow from s" + 2024 str( i ) ) 2025 LossInPings = main.Mininet2.checkForLoss( 2026 "/tmp/ping.h" + 2027 str( i ) ) or LossInPings 2028 if LossInPings == main.TRUE: 2029 main.log.info( "Loss in ping detected" ) 2030 elif LossInPings == main.ERROR: 2031 main.log.info( "There are multiple mininet process running" ) 2032 elif LossInPings == main.FALSE: 2033 main.log.info( "No Loss in the pings" ) 2034 main.log.info( "No loss of dataplane connectivity" ) 2035 utilities.assert_equals( 2036 expect=main.FALSE, 2037 actual=LossInPings, 2038 onpass="No Loss of connectivity", 2039 onfail="Loss of dataplane connectivity detected" ) 2040 ''' 2041 2042 main.step( "Leadership Election is still functional" ) 2043 # Test of LeadershipElection 2044 leaderList = [] 2045 # FIXME: make sure this matches nodes that were restarted 2046 restarted = [ main.nodes[0].ip_address, main.nodes[1].ip_address, 2047 main.nodes[2].ip_address ] 2048 2049 leaderResult = main.TRUE 2050 for cli in main.CLIs: 2051 leaderN = cli.electionTestLeader() 2052 leaderList.append( leaderN ) 2053 if leaderN == main.FALSE: 2054 # error in response 2055 main.log.error( "Something is wrong with " + 2056 "electionTestLeader function, check the" + 2057 " error logs" ) 2058 leaderResult = main.FALSE 2059 elif leaderN is None: 2060 main.log.error( cli.name + 2061 " shows no leader for the election-app was" + 2062 " elected after the old one died" ) 2063 leaderResult = main.FALSE 2064 elif leaderN in restarted: 2065 main.log.error( cli.name + " shows " + str( leaderN ) + 2066 " as leader for the election-app, but it " + 2067 "was restarted" ) 2068 leaderResult = main.FALSE 2069 if len( set( leaderList ) ) != 1: 2070 leaderResult = main.FALSE 2071 main.log.error( 2072 "Inconsistent view of leader for the election test app" ) 2073 # TODO: print the list 2074 utilities.assert_equals( 2075 expect=main.TRUE, 2076 actual=leaderResult, 2077 onpass="Leadership election passed", 2078 onfail="Something went wrong with Leadership election" )
2079
2080 - def CASE8( self, main ):
2081 """ 2082 Compare topo 2083 """ 2084 import json 2085 import time 2086 assert main.numCtrls, "main.numCtrls not defined" 2087 assert main, "main not defined" 2088 assert utilities.assert_equals, "utilities.assert_equals not defined" 2089 assert main.CLIs, "main.CLIs not defined" 2090 assert main.nodes, "main.nodes not defined" 2091 2092 main.case( "Compare ONOS Topology view to Mininet topology" ) 2093 main.caseExplanation = "Compare topology objects between Mininet" +\ 2094 " and ONOS" 2095 2096 main.step( "Comparing ONOS topology to MN" ) 2097 devicesResults = main.TRUE 2098 linksResults = main.TRUE 2099 hostsResults = main.TRUE 2100 hostAttachmentResults = True 2101 topoResult = main.FALSE 2102 elapsed = 0 2103 count = 0 2104 main.step( "Collecting topology information from ONOS" ) 2105 startTime = time.time() 2106 # Give time for Gossip to work 2107 while topoResult == main.FALSE and elapsed < 60: 2108 count += 1 2109 cliStart = time.time() 2110 devices = [] 2111 threads = [] 2112 for i in range( main.numCtrls ): 2113 t = main.Thread( target=main.CLIs[i].devices, 2114 name="devices-" + str( i ), 2115 args=[ ] ) 2116 threads.append( t ) 2117 t.start() 2118 2119 for t in threads: 2120 t.join() 2121 devices.append( t.result ) 2122 hosts = [] 2123 ipResult = main.TRUE 2124 threads = [] 2125 for i in range( main.numCtrls ): 2126 t = main.Thread( target=main.CLIs[i].hosts, 2127 name="hosts-" + str( i ), 2128 args=[ ] ) 2129 threads.append( t ) 2130 t.start() 2131 2132 for t in threads: 2133 t.join() 2134 try: 2135 hosts.append( json.loads( t.result ) ) 2136 except ( ValueError, TypeError ): 2137 main.log.exception( "Error parsing hosts results" ) 2138 main.log.error( repr( t.result ) ) 2139 for controller in range( 0, len( hosts ) ): 2140 controllerStr = str( controller + 1 ) 2141 for host in hosts[ controller ]: 2142 if host is None or host.get( 'ipAddresses', [] ) == []: 2143 main.log.error( 2144 "DEBUG:Error with host ipAddresses on controller" + 2145 controllerStr + ": " + str( host ) ) 2146 ipResult = main.FALSE 2147 ports = [] 2148 threads = [] 2149 for i in range( main.numCtrls ): 2150 t = main.Thread( target=main.CLIs[i].ports, 2151 name="ports-" + str( i ), 2152 args=[ ] ) 2153 threads.append( t ) 2154 t.start() 2155 2156 for t in threads: 2157 t.join() 2158 ports.append( t.result ) 2159 links = [] 2160 threads = [] 2161 for i in range( main.numCtrls ): 2162 t = main.Thread( target=main.CLIs[i].links, 2163 name="links-" + str( i ), 2164 args=[ ] ) 2165 threads.append( t ) 2166 t.start() 2167 2168 for t in threads: 2169 t.join() 2170 links.append( t.result ) 2171 clusters = [] 2172 threads = [] 2173 for i in range( main.numCtrls ): 2174 t = main.Thread( target=main.CLIs[i].clusters, 2175 name="clusters-" + str( i ), 2176 args=[ ] ) 2177 threads.append( t ) 2178 t.start() 2179 2180 for t in threads: 2181 t.join() 2182 clusters.append( t.result ) 2183 2184 elapsed = time.time() - startTime 2185 cliTime = time.time() - cliStart 2186 print "Elapsed time: " + str( elapsed ) 2187 print "CLI time: " + str( cliTime ) 2188 2189 mnSwitches = main.Mininet1.getSwitches() 2190 mnLinks = main.Mininet1.getLinks() 2191 mnHosts = main.Mininet1.getHosts() 2192 for controller in range( main.numCtrls ): 2193 controllerStr = str( controller + 1 ) 2194 if devices[ controller ] and ports[ controller ] and\ 2195 "Error" not in devices[ controller ] and\ 2196 "Error" not in ports[ controller ]: 2197 2198 currentDevicesResult = main.Mininet1.compareSwitches( 2199 mnSwitches, 2200 json.loads( devices[ controller ] ), 2201 json.loads( ports[ controller ] ) ) 2202 else: 2203 currentDevicesResult = main.FALSE 2204 utilities.assert_equals( expect=main.TRUE, 2205 actual=currentDevicesResult, 2206 onpass="ONOS" + controllerStr + 2207 " Switches view is correct", 2208 onfail="ONOS" + controllerStr + 2209 " Switches view is incorrect" ) 2210 2211 if links[ controller ] and "Error" not in links[ controller ]: 2212 currentLinksResult = main.Mininet1.compareLinks( 2213 mnSwitches, mnLinks, 2214 json.loads( links[ controller ] ) ) 2215 else: 2216 currentLinksResult = main.FALSE 2217 utilities.assert_equals( expect=main.TRUE, 2218 actual=currentLinksResult, 2219 onpass="ONOS" + controllerStr + 2220 " links view is correct", 2221 onfail="ONOS" + controllerStr + 2222 " links view is incorrect" ) 2223 2224 if hosts[ controller ] or "Error" not in hosts[ controller ]: 2225 currentHostsResult = main.Mininet1.compareHosts( 2226 mnHosts, 2227 hosts[ controller ] ) 2228 else: 2229 currentHostsResult = main.FALSE 2230 utilities.assert_equals( expect=main.TRUE, 2231 actual=currentHostsResult, 2232 onpass="ONOS" + controllerStr + 2233 " hosts exist in Mininet", 2234 onfail="ONOS" + controllerStr + 2235 " hosts don't match Mininet" ) 2236 # CHECKING HOST ATTACHMENT POINTS 2237 hostAttachment = True 2238 zeroHosts = False 2239 # FIXME: topo-HA/obelisk specific mappings: 2240 # key is mac and value is dpid 2241 mappings = {} 2242 for i in range( 1, 29 ): # hosts 1 through 28 2243 # set up correct variables: 2244 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2) 2245 if i == 1: 2246 deviceId = "1000".zfill(16) 2247 elif i == 2: 2248 deviceId = "2000".zfill(16) 2249 elif i == 3: 2250 deviceId = "3000".zfill(16) 2251 elif i == 4: 2252 deviceId = "3004".zfill(16) 2253 elif i == 5: 2254 deviceId = "5000".zfill(16) 2255 elif i == 6: 2256 deviceId = "6000".zfill(16) 2257 elif i == 7: 2258 deviceId = "6007".zfill(16) 2259 elif i >= 8 and i <= 17: 2260 dpid = '3' + str( i ).zfill( 3 ) 2261 deviceId = dpid.zfill(16) 2262 elif i >= 18 and i <= 27: 2263 dpid = '6' + str( i ).zfill( 3 ) 2264 deviceId = dpid.zfill(16) 2265 elif i == 28: 2266 deviceId = "2800".zfill(16) 2267 mappings[ macId ] = deviceId 2268 if hosts[ controller ] or "Error" not in hosts[ controller ]: 2269 if hosts[ controller ] == []: 2270 main.log.warn( "There are no hosts discovered" ) 2271 zeroHosts = True 2272 else: 2273 for host in hosts[ controller ]: 2274 mac = None 2275 location = None 2276 device = None 2277 port = None 2278 try: 2279 mac = host.get( 'mac' ) 2280 assert mac, "mac field could not be found for this host object" 2281 2282 location = host.get( 'location' ) 2283 assert location, "location field could not be found for this host object" 2284 2285 # Trim the protocol identifier off deviceId 2286 device = str( location.get( 'elementId' ) ).split(':')[1] 2287 assert device, "elementId field could not be found for this host location object" 2288 2289 port = location.get( 'port' ) 2290 assert port, "port field could not be found for this host location object" 2291 2292 # Now check if this matches where they should be 2293 if mac and device and port: 2294 if str( port ) != "1": 2295 main.log.error( "The attachment port is incorrect for " + 2296 "host " + str( mac ) + 2297 ". Expected: 1 Actual: " + str( port) ) 2298 hostAttachment = False 2299 if device != mappings[ str( mac ) ]: 2300 main.log.error( "The attachment device is incorrect for " + 2301 "host " + str( mac ) + 2302 ". Expected: " + mappings[ str( mac ) ] + 2303 " Actual: " + device ) 2304 hostAttachment = False 2305 else: 2306 hostAttachment = False 2307 except AssertionError: 2308 main.log.exception( "Json object not as expected" ) 2309 main.log.error( repr( host ) ) 2310 hostAttachment = False 2311 else: 2312 main.log.error( "No hosts json output or \"Error\"" + 2313 " in output. hosts = " + 2314 repr( hosts[ controller ] ) ) 2315 if zeroHosts is False: 2316 hostAttachment = True 2317 2318 # END CHECKING HOST ATTACHMENT POINTS 2319 devicesResults = devicesResults and currentDevicesResult 2320 linksResults = linksResults and currentLinksResult 2321 hostsResults = hostsResults and currentHostsResult 2322 hostAttachmentResults = hostAttachmentResults and\ 2323 hostAttachment 2324 2325 # Compare json objects for hosts and dataplane clusters 2326 2327 # hosts 2328 main.step( "Hosts view is consistent across all ONOS nodes" ) 2329 consistentHostsResult = main.TRUE 2330 for controller in range( len( hosts ) ): 2331 controllerStr = str( controller + 1 ) 2332 if "Error" not in hosts[ controller ]: 2333 if hosts[ controller ] == hosts[ 0 ]: 2334 continue 2335 else: # hosts not consistent 2336 main.log.error( "hosts from ONOS" + controllerStr + 2337 " is inconsistent with ONOS1" ) 2338 main.log.warn( repr( hosts[ controller ] ) ) 2339 consistentHostsResult = main.FALSE 2340 2341 else: 2342 main.log.error( "Error in getting ONOS hosts from ONOS" + 2343 controllerStr ) 2344 consistentHostsResult = main.FALSE 2345 main.log.warn( "ONOS" + controllerStr + 2346 " hosts response: " + 2347 repr( hosts[ controller ] ) ) 2348 utilities.assert_equals( 2349 expect=main.TRUE, 2350 actual=consistentHostsResult, 2351 onpass="Hosts view is consistent across all ONOS nodes", 2352 onfail="ONOS nodes have different views of hosts" ) 2353 2354 main.step( "Hosts information is correct" ) 2355 hostsResults = hostsResults and ipResult 2356 utilities.assert_equals( 2357 expect=main.TRUE, 2358 actual=hostsResults, 2359 onpass="Host information is correct", 2360 onfail="Host information is incorrect" ) 2361 2362 main.step( "Host attachment points to the network" ) 2363 utilities.assert_equals( 2364 expect=True, 2365 actual=hostAttachmentResults, 2366 onpass="Hosts are correctly attached to the network", 2367 onfail="ONOS did not correctly attach hosts to the network" ) 2368 2369 # Strongly connected clusters of devices 2370 main.step( "Clusters view is consistent across all ONOS nodes" ) 2371 consistentClustersResult = main.TRUE 2372 for controller in range( len( clusters ) ): 2373 controllerStr = str( controller + 1 ) 2374 if "Error" not in clusters[ controller ]: 2375 if clusters[ controller ] == clusters[ 0 ]: 2376 continue 2377 else: # clusters not consistent 2378 main.log.error( "clusters from ONOS" + 2379 controllerStr + 2380 " is inconsistent with ONOS1" ) 2381 consistentClustersResult = main.FALSE 2382 2383 else: 2384 main.log.error( "Error in getting dataplane clusters " + 2385 "from ONOS" + controllerStr ) 2386 consistentClustersResult = main.FALSE 2387 main.log.warn( "ONOS" + controllerStr + 2388 " clusters response: " + 2389 repr( clusters[ controller ] ) ) 2390 utilities.assert_equals( 2391 expect=main.TRUE, 2392 actual=consistentClustersResult, 2393 onpass="Clusters view is consistent across all ONOS nodes", 2394 onfail="ONOS nodes have different views of clusters" ) 2395 2396 main.step( "There is only one SCC" ) 2397 # there should always only be one cluster 2398 try: 2399 numClusters = len( json.loads( clusters[ 0 ] ) ) 2400 except ( ValueError, TypeError ): 2401 main.log.exception( "Error parsing clusters[0]: " + 2402 repr( clusters[0] ) ) 2403 clusterResults = main.FALSE 2404 if numClusters == 1: 2405 clusterResults = main.TRUE 2406 utilities.assert_equals( 2407 expect=1, 2408 actual=numClusters, 2409 onpass="ONOS shows 1 SCC", 2410 onfail="ONOS shows " + str( numClusters ) + " SCCs" ) 2411 2412 topoResult = ( devicesResults and linksResults 2413 and hostsResults and consistentHostsResult 2414 and consistentClustersResult and clusterResults 2415 and ipResult and hostAttachmentResults ) 2416 2417 topoResult = topoResult and int( count <= 2 ) 2418 note = "note it takes about " + str( int( cliTime ) ) + \ 2419 " seconds for the test to make all the cli calls to fetch " +\ 2420 "the topology from each ONOS instance" 2421 main.log.info( 2422 "Very crass estimate for topology discovery/convergence( " + 2423 str( note ) + " ): " + str( elapsed ) + " seconds, " + 2424 str( count ) + " tries" ) 2425 2426 main.step( "Device information is correct" ) 2427 utilities.assert_equals( 2428 expect=main.TRUE, 2429 actual=devicesResults, 2430 onpass="Device information is correct", 2431 onfail="Device information is incorrect" ) 2432 2433 main.step( "Links are correct" ) 2434 utilities.assert_equals( 2435 expect=main.TRUE, 2436 actual=linksResults, 2437 onpass="Link are correct", 2438 onfail="Links are incorrect" ) 2439 2440 # FIXME: move this to an ONOS state case 2441 main.step( "Checking ONOS nodes" ) 2442 nodesOutput = [] 2443 nodeResults = main.TRUE 2444 threads = [] 2445 for i in range( main.numCtrls ): 2446 t = main.Thread( target=main.CLIs[i].nodes, 2447 name="nodes-" + str( i ), 2448 args=[ ] ) 2449 threads.append( t ) 2450 t.start() 2451 2452 for t in threads: 2453 t.join() 2454 nodesOutput.append( t.result ) 2455 ips = [ node.ip_address for node in main.nodes ] 2456 for i in nodesOutput: 2457 try: 2458 current = json.loads( i ) 2459 for node in current: 2460 currentResult = main.FALSE 2461 if node['ip'] in ips: # node in nodes() output is in cell 2462 if node['state'] == 'ACTIVE': 2463 currentResult = main.TRUE 2464 else: 2465 main.log.error( "Error in ONOS node availability" ) 2466 main.log.error( 2467 json.dumps( current, 2468 sort_keys=True, 2469 indent=4, 2470 separators=( ',', ': ' ) ) ) 2471 break 2472 nodeResults = nodeResults and currentResult 2473 except ( ValueError, TypeError ): 2474 main.log.error( "Error parsing nodes output" ) 2475 main.log.warn( repr( i ) ) 2476 utilities.assert_equals( expect=main.TRUE, actual=nodeResults, 2477 onpass="Nodes check successful", 2478 onfail="Nodes check NOT successful" )
2479
2480 - def CASE9( self, main ):
2481 """ 2482 Link s3-s28 down 2483 """ 2484 import time 2485 assert main.numCtrls, "main.numCtrls not defined" 2486 assert main, "main not defined" 2487 assert utilities.assert_equals, "utilities.assert_equals not defined" 2488 assert main.CLIs, "main.CLIs not defined" 2489 assert main.nodes, "main.nodes not defined" 2490 # NOTE: You should probably run a topology check after this 2491 2492 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] ) 2493 2494 description = "Turn off a link to ensure that Link Discovery " +\ 2495 "is working properly" 2496 main.case( description ) 2497 2498 main.step( "Kill Link between s3 and s28" ) 2499 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" ) 2500 main.log.info( "Waiting " + str( linkSleep ) + 2501 " seconds for link down to be discovered" ) 2502 time.sleep( linkSleep ) 2503 utilities.assert_equals( expect=main.TRUE, actual=LinkDown, 2504 onpass="Link down successful", 2505 onfail="Failed to bring link down" )
2506 # TODO do some sort of check here 2507
2508 - def CASE10( self, main ):
2509 """ 2510 Link s3-s28 up 2511 """ 2512 import time 2513 assert main.numCtrls, "main.numCtrls not defined" 2514 assert main, "main not defined" 2515 assert utilities.assert_equals, "utilities.assert_equals not defined" 2516 assert main.CLIs, "main.CLIs not defined" 2517 assert main.nodes, "main.nodes not defined" 2518 # NOTE: You should probably run a topology check after this 2519 2520 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] ) 2521 2522 description = "Restore a link to ensure that Link Discovery is " + \ 2523 "working properly" 2524 main.case( description ) 2525 2526 main.step( "Bring link between s3 and s28 back up" ) 2527 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" ) 2528 main.log.info( "Waiting " + str( linkSleep ) + 2529 " seconds for link up to be discovered" ) 2530 time.sleep( linkSleep ) 2531 utilities.assert_equals( expect=main.TRUE, actual=LinkUp, 2532 onpass="Link up successful", 2533 onfail="Failed to bring link up" )
2534 # TODO do some sort of check here 2535
2536 - def CASE11( self, main ):
2537 """ 2538 Switch Down 2539 """ 2540 # NOTE: You should probably run a topology check after this 2541 import time 2542 assert main.numCtrls, "main.numCtrls not defined" 2543 assert main, "main not defined" 2544 assert utilities.assert_equals, "utilities.assert_equals not defined" 2545 assert main.CLIs, "main.CLIs not defined" 2546 assert main.nodes, "main.nodes not defined" 2547 2548 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) 2549 2550 description = "Killing a switch to ensure it is discovered correctly" 2551 main.case( description ) 2552 switch = main.params[ 'kill' ][ 'switch' ] 2553 switchDPID = main.params[ 'kill' ][ 'dpid' ] 2554 2555 # TODO: Make this switch parameterizable 2556 main.step( "Kill " + switch ) 2557 main.log.info( "Deleting " + switch ) 2558 main.Mininet1.delSwitch( switch ) 2559 main.log.info( "Waiting " + str( switchSleep ) + 2560 " seconds for switch down to be discovered" ) 2561 time.sleep( switchSleep ) 2562 device = main.ONOScli1.getDevice( dpid=switchDPID ) 2563 # Peek at the deleted switch 2564 main.log.warn( str( device ) ) 2565 result = main.FALSE 2566 if device and device[ 'available' ] is False: 2567 result = main.TRUE 2568 utilities.assert_equals( expect=main.TRUE, actual=result, 2569 onpass="Kill switch successful", 2570 onfail="Failed to kill switch?" )
2571
2572 - def CASE12( self, main ):
2573 """ 2574 Switch Up 2575 """ 2576 # NOTE: You should probably run a topology check after this 2577 import time 2578 assert main.numCtrls, "main.numCtrls not defined" 2579 assert main, "main not defined" 2580 assert utilities.assert_equals, "utilities.assert_equals not defined" 2581 assert main.CLIs, "main.CLIs not defined" 2582 assert main.nodes, "main.nodes not defined" 2583 assert ONOS1Port, "ONOS1Port not defined" 2584 assert ONOS2Port, "ONOS2Port not defined" 2585 assert ONOS3Port, "ONOS3Port not defined" 2586 assert ONOS4Port, "ONOS4Port not defined" 2587 assert ONOS5Port, "ONOS5Port not defined" 2588 assert ONOS6Port, "ONOS6Port not defined" 2589 assert ONOS7Port, "ONOS7Port not defined" 2590 2591 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) 2592 switch = main.params[ 'kill' ][ 'switch' ] 2593 switchDPID = main.params[ 'kill' ][ 'dpid' ] 2594 links = main.params[ 'kill' ][ 'links' ].split() 2595 description = "Adding a switch to ensure it is discovered correctly" 2596 main.case( description ) 2597 2598 main.step( "Add back " + switch ) 2599 main.Mininet1.addSwitch( switch, dpid=switchDPID ) 2600 for peer in links: 2601 main.Mininet1.addLink( switch, peer ) 2602 ipList = [] 2603 for i in range( main.numCtrls ): 2604 ipList.append( main.nodes[ i ].ip_address ) 2605 main.Mininet1.assignSwController( sw=switch, ip=ipList ) 2606 main.log.info( "Waiting " + str( switchSleep ) + 2607 " seconds for switch up to be discovered" ) 2608 time.sleep( switchSleep ) 2609 device = main.ONOScli1.getDevice( dpid=switchDPID ) 2610 # Peek at the deleted switch 2611 main.log.warn( str( device ) ) 2612 result = main.FALSE 2613 if device and device[ 'available' ]: 2614 result = main.TRUE 2615 utilities.assert_equals( expect=main.TRUE, actual=result, 2616 onpass="add switch successful", 2617 onfail="Failed to add switch?" )
2618
2619 - def CASE13( self, main ):
2620 """ 2621 Clean up 2622 """ 2623 import os 2624 import time 2625 assert main.numCtrls, "main.numCtrls not defined" 2626 assert main, "main not defined" 2627 assert utilities.assert_equals, "utilities.assert_equals not defined" 2628 assert main.CLIs, "main.CLIs not defined" 2629 assert main.nodes, "main.nodes not defined" 2630 2631 # printing colors to terminal 2632 colors = { 'cyan': '\033[96m', 'purple': '\033[95m', 2633 'blue': '\033[94m', 'green': '\033[92m', 2634 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' } 2635 main.case( "Test Cleanup" ) 2636 main.step( "Killing tcpdumps" ) 2637 main.Mininet2.stopTcpdump() 2638 2639 testname = main.TEST 2640 if main.params[ 'BACKUP' ] == "True": 2641 main.step( "Copying MN pcap and ONOS log files to test station" ) 2642 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ] 2643 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ] 2644 # NOTE: MN Pcap file is being saved to ~/packet_captures 2645 # scp this file as MN and TestON aren't necessarily the same vm 2646 # FIXME: scp 2647 # mn files 2648 # TODO: Load these from params 2649 # NOTE: must end in / 2650 logFolder = "/opt/onos/log/" 2651 logFiles = [ "karaf.log", "karaf.log.1" ] 2652 # NOTE: must end in / 2653 dstDir = "~/packet_captures/" 2654 for f in logFiles: 2655 for node in main.nodes: 2656 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address + 2657 ":" + logFolder + f + " " + 2658 teststationUser + "@" + 2659 teststationIP + ":" + 2660 dstDir + str( testname ) + 2661 "-" + node.name + "-" + f ) 2662 main.ONOSbench.handle.expect( "\$" ) 2663 2664 # std*.log's 2665 # NOTE: must end in / 2666 logFolder = "/opt/onos/var/" 2667 logFiles = [ "stderr.log", "stdout.log" ] 2668 # NOTE: must end in / 2669 dstDir = "~/packet_captures/" 2670 for f in logFiles: 2671 for node in main.nodes: 2672 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address + 2673 ":" + logFolder + f + " " + 2674 teststationUser + "@" + 2675 teststationIP + ":" + 2676 dstDir + str( testname ) + 2677 "-" + node.name + "-" + f ) 2678 main.ONOSbench.handle.expect( "\$" ) 2679 # sleep so scp can finish 2680 time.sleep( 10 ) 2681 main.step( "Packing and rotating pcap archives" ) 2682 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) ) 2683 2684 main.step( "Stopping Mininet" ) 2685 mnResult = main.Mininet1.stopNet() 2686 utilities.assert_equals( expect=main.TRUE, actual=mnResult, 2687 onpass="Mininet stopped", 2688 onfail="MN cleanup NOT successful" ) 2689 2690 main.step( "Checking ONOS Logs for errors" ) 2691 for node in main.nodes: 2692 print colors[ 'purple' ] + "Checking logs for errors on " + \ 2693 node.name + ":" + colors[ 'end' ] 2694 print main.ONOSbench.checkLogs( node.ip_address, restart=True ) 2695 2696 try: 2697 timerLog = open( main.logdir + "/Timers.csv", 'w') 2698 # Overwrite with empty line and close 2699 labels = "Gossip Intents, Restart" 2700 data = str( gossipTime ) + ", " + str( main.restartTime ) 2701 timerLog.write( labels + "\n" + data ) 2702 timerLog.close() 2703 except NameError, e: 2704 main.log.exception(e)
2705
2706 - def CASE14( self, main ):
2707 """ 2708 start election app on all onos nodes 2709 """ 2710 assert main.numCtrls, "main.numCtrls not defined" 2711 assert main, "main not defined" 2712 assert utilities.assert_equals, "utilities.assert_equals not defined" 2713 assert main.CLIs, "main.CLIs not defined" 2714 assert main.nodes, "main.nodes not defined" 2715 2716 main.case("Start Leadership Election app") 2717 main.step( "Install leadership election app" ) 2718 appResult = main.ONOScli1.activateApp( "org.onosproject.election" ) 2719 utilities.assert_equals( 2720 expect=main.TRUE, 2721 actual=appResult, 2722 onpass="Election app installed", 2723 onfail="Something went wrong with installing Leadership election" ) 2724 2725 main.step( "Run for election on each node" ) 2726 leaderResult = main.TRUE 2727 leaders = [] 2728 for cli in main.CLIs: 2729 cli.electionTestRun() 2730 for cli in main.CLIs: 2731 leader = cli.electionTestLeader() 2732 if leader is None or leader == main.FALSE: 2733 main.log.error( cli.name + ": Leader for the election app " + 2734 "should be an ONOS node, instead got '" + 2735 str( leader ) + "'" ) 2736 leaderResult = main.FALSE 2737 leaders.append( leader ) 2738 utilities.assert_equals( 2739 expect=main.TRUE, 2740 actual=leaderResult, 2741 onpass="Successfully ran for leadership", 2742 onfail="Failed to run for leadership" ) 2743 2744 main.step( "Check that each node shows the same leader" ) 2745 sameLeader = main.TRUE 2746 if len( set( leaders ) ) != 1: 2747 sameLeader = main.FALSE 2748 main.log.error( "Results of electionTestLeader is order of main.CLIs:" + 2749 str( leaders ) ) 2750 utilities.assert_equals( 2751 expect=main.TRUE, 2752 actual=sameLeader, 2753 onpass="Leadership is consistent for the election topic", 2754 onfail="Nodes have different leaders" )
2755
2756 - def CASE15( self, main ):
2757 """ 2758 Check that Leadership Election is still functional 2759 """ 2760 import time 2761 assert main.numCtrls, "main.numCtrls not defined" 2762 assert main, "main not defined" 2763 assert utilities.assert_equals, "utilities.assert_equals not defined" 2764 assert main.CLIs, "main.CLIs not defined" 2765 assert main.nodes, "main.nodes not defined" 2766 2767 leaderResult = main.TRUE 2768 description = "Check that Leadership Election is still functional" 2769 main.case( description ) 2770 2771 main.step( "Check that each node shows the same leader" ) 2772 sameLeader = main.TRUE 2773 leaders = [] 2774 for cli in main.CLIs: 2775 leader = cli.electionTestLeader() 2776 leaders.append( leader ) 2777 if len( set( leaders ) ) != 1: 2778 sameLeader = main.FALSE 2779 main.log.error( "Results of electionTestLeader is order of main.CLIs:" + 2780 str( leaders ) ) 2781 utilities.assert_equals( 2782 expect=main.TRUE, 2783 actual=sameLeader, 2784 onpass="Leadership is consistent for the election topic", 2785 onfail="Nodes have different leaders" ) 2786 2787 main.step( "Find current leader and withdraw" ) 2788 leader = main.ONOScli1.electionTestLeader() 2789 # do some sanity checking on leader before using it 2790 withdrawResult = main.FALSE 2791 if leader is None or leader == main.FALSE: 2792 main.log.error( 2793 "Leader for the election app should be an ONOS node," + 2794 "instead got '" + str( leader ) + "'" ) 2795 leaderResult = main.FALSE 2796 oldLeader = None 2797 for i in range( len( main.CLIs ) ): 2798 if leader == main.nodes[ i ].ip_address: 2799 oldLeader = main.CLIs[ i ] 2800 break 2801 else: # FOR/ELSE statement 2802 main.log.error( "Leader election, could not find current leader" ) 2803 if oldLeader: 2804 withdrawResult = oldLeader.electionTestWithdraw() 2805 utilities.assert_equals( 2806 expect=main.TRUE, 2807 actual=withdrawResult, 2808 onpass="Node was withdrawn from election", 2809 onfail="Node was not withdrawn from election" ) 2810 2811 main.step( "Make sure new leader is elected" ) 2812 # FIXME: use threads 2813 leaderList = [] 2814 for cli in main.CLIs: 2815 leaderN = cli.electionTestLeader() 2816 leaderList.append( leaderN ) 2817 if leaderN == leader: 2818 main.log.error( cli.name + " still sees " + str( leader ) + 2819 " as leader after they withdrew" ) 2820 leaderResult = main.FALSE 2821 elif leaderN == main.FALSE: 2822 # error in response 2823 # TODO: add check for "Command not found:" in the driver, this 2824 # means the app isn't loaded 2825 main.log.error( "Something is wrong with " + 2826 "electionTestLeader function, " + 2827 "check the error logs" ) 2828 leaderResult = main.FALSE 2829 elif leaderN is None: 2830 # node may not have recieved the event yet 2831 time.sleep(7) 2832 leaderN = cli.electionTestLeader() 2833 leaderList.pop() 2834 leaderList.append( leaderN ) 2835 consistentLeader = main.FALSE 2836 if len( set( leaderList ) ) == 1: 2837 main.log.info( "Each Election-app sees '" + 2838 str( leaderList[ 0 ] ) + 2839 "' as the leader" ) 2840 consistentLeader = main.TRUE 2841 else: 2842 main.log.error( 2843 "Inconsistent responses for leader of Election-app:" ) 2844 for n in range( len( leaderList ) ): 2845 main.log.error( "ONOS" + str( n + 1 ) + " response: " + 2846 str( leaderList[ n ] ) ) 2847 leaderResult = leaderResult and consistentLeader 2848 utilities.assert_equals( 2849 expect=main.TRUE, 2850 actual=leaderResult, 2851 onpass="Leadership election passed", 2852 onfail="Something went wrong with Leadership election" ) 2853 2854 main.step( "Run for election on old leader( just so everyone " + 2855 "is in the hat )" ) 2856 if oldLeader: 2857 runResult = oldLeader.electionTestRun() 2858 else: 2859 runResult = main.FALSE 2860 utilities.assert_equals( 2861 expect=main.TRUE, 2862 actual=runResult, 2863 onpass="App re-ran for election", 2864 onfail="App failed to run for election" ) 2865 2866 main.step( "Leader did not change when old leader re-ran" ) 2867 afterRun = main.ONOScli1.electionTestLeader() 2868 # verify leader didn't just change 2869 if afterRun == leaderList[ 0 ]: 2870 afterResult = main.TRUE 2871 else: 2872 afterResult = main.FALSE 2873 2874 utilities.assert_equals( 2875 expect=main.TRUE, 2876 actual=afterResult, 2877 onpass="Old leader successfully re-ran for election", 2878 onfail="Something went wrong with Leadership election after " + 2879 "the old leader re-ran for election" )
2880
2881 - def CASE16( self, main ):
2882 """ 2883 Install Distributed Primitives app 2884 """ 2885 import time 2886 assert main.numCtrls, "main.numCtrls not defined" 2887 assert main, "main not defined" 2888 assert utilities.assert_equals, "utilities.assert_equals not defined" 2889 assert main.CLIs, "main.CLIs not defined" 2890 assert main.nodes, "main.nodes not defined" 2891 2892 # Variables for the distributed primitives tests 2893 global pCounterName 2894 global iCounterName 2895 global pCounterValue 2896 global iCounterValue 2897 global onosSet 2898 global onosSetName 2899 pCounterName = "TestON-Partitions" 2900 iCounterName = "TestON-inMemory" 2901 pCounterValue = 0 2902 iCounterValue = 0 2903 onosSet = set([]) 2904 onosSetName = "TestON-set" 2905 2906 description = "Install Primitives app" 2907 main.case( description ) 2908 main.step( "Install Primitives app" ) 2909 appName = "org.onosproject.distributedprimitives" 2910 appResults = main.CLIs[0].activateApp( appName ) 2911 utilities.assert_equals( expect=main.TRUE, 2912 actual=appResults, 2913 onpass="Primitives app activated", 2914 onfail="Primitives app not activated" ) 2915 time.sleep( 5 ) # To allow all nodes to activate
2916
2917 - def CASE17( self, main ):
2918 """ 2919 Check for basic functionality with distributed primitives 2920 """ 2921 # Make sure variables are defined/set 2922 assert main.numCtrls, "main.numCtrls not defined" 2923 assert main, "main not defined" 2924 assert utilities.assert_equals, "utilities.assert_equals not defined" 2925 assert main.CLIs, "main.CLIs not defined" 2926 assert main.nodes, "main.nodes not defined" 2927 assert pCounterName, "pCounterName not defined" 2928 assert iCounterName, "iCounterName not defined" 2929 assert onosSetName, "onosSetName not defined" 2930 # NOTE: assert fails if value is 0/None/Empty/False 2931 try: 2932 pCounterValue 2933 except NameError: 2934 main.log.error( "pCounterValue not defined, setting to 0" ) 2935 pCounterValue = 0 2936 try: 2937 iCounterValue 2938 except NameError: 2939 main.log.error( "iCounterValue not defined, setting to 0" ) 2940 iCounterValue = 0 2941 try: 2942 onosSet 2943 except NameError: 2944 main.log.error( "onosSet not defined, setting to empty Set" ) 2945 onosSet = set([]) 2946 # Variables for the distributed primitives tests. These are local only 2947 addValue = "a" 2948 addAllValue = "a b c d e f" 2949 retainValue = "c d e f" 2950 2951 description = "Check for basic functionality with distributed " +\ 2952 "primitives" 2953 main.case( description ) 2954 main.caseExplanation = "Test the methods of the distributed " +\ 2955 "primitives (counters and sets) throught the cli" 2956 # DISTRIBUTED ATOMIC COUNTERS 2957 # Partitioned counters 2958 main.step( "Increment then get a default counter on each node" ) 2959 pCounters = [] 2960 threads = [] 2961 addedPValues = [] 2962 for i in range( main.numCtrls ): 2963 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 2964 name="counterAddAndGet-" + str( i ), 2965 args=[ pCounterName ] ) 2966 pCounterValue += 1 2967 addedPValues.append( pCounterValue ) 2968 threads.append( t ) 2969 t.start() 2970 2971 for t in threads: 2972 t.join() 2973 pCounters.append( t.result ) 2974 # Check that counter incremented numController times 2975 pCounterResults = True 2976 for i in addedPValues: 2977 tmpResult = i in pCounters 2978 pCounterResults = pCounterResults and tmpResult 2979 if not tmpResult: 2980 main.log.error( str( i ) + " is not in partitioned " 2981 "counter incremented results" ) 2982 utilities.assert_equals( expect=True, 2983 actual=pCounterResults, 2984 onpass="Default counter incremented", 2985 onfail="Error incrementing default" + 2986 " counter" ) 2987 2988 main.step( "Get then Increment a default counter on each node" ) 2989 pCounters = [] 2990 threads = [] 2991 addedPValues = [] 2992 for i in range( main.numCtrls ): 2993 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 2994 name="counterGetAndAdd-" + str( i ), 2995 args=[ pCounterName ] ) 2996 addedPValues.append( pCounterValue ) 2997 pCounterValue += 1 2998 threads.append( t ) 2999 t.start() 3000 3001 for t in threads: 3002 t.join() 3003 pCounters.append( t.result ) 3004 # Check that counter incremented numController times 3005 pCounterResults = True 3006 for i in addedPValues: 3007 tmpResult = i in pCounters 3008 pCounterResults = pCounterResults and tmpResult 3009 if not tmpResult: 3010 main.log.error( str( i ) + " is not in partitioned " 3011 "counter incremented results" ) 3012 utilities.assert_equals( expect=True, 3013 actual=pCounterResults, 3014 onpass="Default counter incremented", 3015 onfail="Error incrementing default" + 3016 " counter" ) 3017 3018 main.step( "Counters we added have the correct values" ) 3019 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue ) 3020 utilities.assert_equals( expect=main.TRUE, 3021 actual=incrementCheck, 3022 onpass="Added counters are correct", 3023 onfail="Added counters are incorrect" ) 3024 3025 main.step( "Add -8 to then get a default counter on each node" ) 3026 pCounters = [] 3027 threads = [] 3028 addedPValues = [] 3029 for i in range( main.numCtrls ): 3030 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3031 name="counterIncrement-" + str( i ), 3032 args=[ pCounterName ], 3033 kwargs={ "delta": -8 } ) 3034 pCounterValue += -8 3035 addedPValues.append( pCounterValue ) 3036 threads.append( t ) 3037 t.start() 3038 3039 for t in threads: 3040 t.join() 3041 pCounters.append( t.result ) 3042 # Check that counter incremented numController times 3043 pCounterResults = True 3044 for i in addedPValues: 3045 tmpResult = i in pCounters 3046 pCounterResults = pCounterResults and tmpResult 3047 if not tmpResult: 3048 main.log.error( str( i ) + " is not in partitioned " 3049 "counter incremented results" ) 3050 utilities.assert_equals( expect=True, 3051 actual=pCounterResults, 3052 onpass="Default counter incremented", 3053 onfail="Error incrementing default" + 3054 " counter" ) 3055 3056 main.step( "Add 5 to then get a default counter on each node" ) 3057 pCounters = [] 3058 threads = [] 3059 addedPValues = [] 3060 for i in range( main.numCtrls ): 3061 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3062 name="counterIncrement-" + str( i ), 3063 args=[ pCounterName ], 3064 kwargs={ "delta": 5 } ) 3065 pCounterValue += 5 3066 addedPValues.append( pCounterValue ) 3067 threads.append( t ) 3068 t.start() 3069 3070 for t in threads: 3071 t.join() 3072 pCounters.append( t.result ) 3073 # Check that counter incremented numController times 3074 pCounterResults = True 3075 for i in addedPValues: 3076 tmpResult = i in pCounters 3077 pCounterResults = pCounterResults and tmpResult 3078 if not tmpResult: 3079 main.log.error( str( i ) + " is not in partitioned " 3080 "counter incremented results" ) 3081 utilities.assert_equals( expect=True, 3082 actual=pCounterResults, 3083 onpass="Default counter incremented", 3084 onfail="Error incrementing default" + 3085 " counter" ) 3086 3087 main.step( "Get then add 5 to a default counter on each node" ) 3088 pCounters = [] 3089 threads = [] 3090 addedPValues = [] 3091 for i in range( main.numCtrls ): 3092 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3093 name="counterIncrement-" + str( i ), 3094 args=[ pCounterName ], 3095 kwargs={ "delta": 5 } ) 3096 addedPValues.append( pCounterValue ) 3097 pCounterValue += 5 3098 threads.append( t ) 3099 t.start() 3100 3101 for t in threads: 3102 t.join() 3103 pCounters.append( t.result ) 3104 # Check that counter incremented numController times 3105 pCounterResults = True 3106 for i in addedPValues: 3107 tmpResult = i in pCounters 3108 pCounterResults = pCounterResults and tmpResult 3109 if not tmpResult: 3110 main.log.error( str( i ) + " is not in partitioned " 3111 "counter incremented results" ) 3112 utilities.assert_equals( expect=True, 3113 actual=pCounterResults, 3114 onpass="Default counter incremented", 3115 onfail="Error incrementing default" + 3116 " counter" ) 3117 3118 main.step( "Counters we added have the correct values" ) 3119 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue ) 3120 utilities.assert_equals( expect=main.TRUE, 3121 actual=incrementCheck, 3122 onpass="Added counters are correct", 3123 onfail="Added counters are incorrect" ) 3124 3125 # In-Memory counters 3126 main.step( "Increment and get an in-memory counter on each node" ) 3127 iCounters = [] 3128 addedIValues = [] 3129 threads = [] 3130 for i in range( main.numCtrls ): 3131 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3132 name="icounterIncrement-" + str( i ), 3133 args=[ iCounterName ], 3134 kwargs={ "inMemory": True } ) 3135 iCounterValue += 1 3136 addedIValues.append( iCounterValue ) 3137 threads.append( t ) 3138 t.start() 3139 3140 for t in threads: 3141 t.join() 3142 iCounters.append( t.result ) 3143 # Check that counter incremented numController times 3144 iCounterResults = True 3145 for i in addedIValues: 3146 tmpResult = i in iCounters 3147 iCounterResults = iCounterResults and tmpResult 3148 if not tmpResult: 3149 main.log.error( str( i ) + " is not in the in-memory " 3150 "counter incremented results" ) 3151 utilities.assert_equals( expect=True, 3152 actual=iCounterResults, 3153 onpass="In-memory counter incremented", 3154 onfail="Error incrementing in-memory" + 3155 " counter" ) 3156 3157 main.step( "Get then Increment a in-memory counter on each node" ) 3158 iCounters = [] 3159 threads = [] 3160 addedIValues = [] 3161 for i in range( main.numCtrls ): 3162 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3163 name="counterGetAndAdd-" + str( i ), 3164 args=[ iCounterName ], 3165 kwargs={ "inMemory": True } ) 3166 addedIValues.append( iCounterValue ) 3167 iCounterValue += 1 3168 threads.append( t ) 3169 t.start() 3170 3171 for t in threads: 3172 t.join() 3173 iCounters.append( t.result ) 3174 # Check that counter incremented numController times 3175 iCounterResults = True 3176 for i in addedIValues: 3177 tmpResult = i in iCounters 3178 iCounterResults = iCounterResults and tmpResult 3179 if not tmpResult: 3180 main.log.error( str( i ) + " is not in in-memory " 3181 "counter incremented results" ) 3182 utilities.assert_equals( expect=True, 3183 actual=iCounterResults, 3184 onpass="In-memory counter incremented", 3185 onfail="Error incrementing in-memory" + 3186 " counter" ) 3187 3188 main.step( "Counters we added have the correct values" ) 3189 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 3190 utilities.assert_equals( expect=main.TRUE, 3191 actual=incrementCheck, 3192 onpass="Added counters are correct", 3193 onfail="Added counters are incorrect" ) 3194 3195 main.step( "Add -8 to then get a in-memory counter on each node" ) 3196 iCounters = [] 3197 threads = [] 3198 addedIValues = [] 3199 for i in range( main.numCtrls ): 3200 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3201 name="counterIncrement-" + str( i ), 3202 args=[ iCounterName ], 3203 kwargs={ "delta": -8, "inMemory": True } ) 3204 iCounterValue += -8 3205 addedIValues.append( iCounterValue ) 3206 threads.append( t ) 3207 t.start() 3208 3209 for t in threads: 3210 t.join() 3211 iCounters.append( t.result ) 3212 # Check that counter incremented numController times 3213 iCounterResults = True 3214 for i in addedIValues: 3215 tmpResult = i in iCounters 3216 iCounterResults = iCounterResults and tmpResult 3217 if not tmpResult: 3218 main.log.error( str( i ) + " is not in in-memory " 3219 "counter incremented results" ) 3220 utilities.assert_equals( expect=True, 3221 actual=pCounterResults, 3222 onpass="In-memory counter incremented", 3223 onfail="Error incrementing in-memory" + 3224 " counter" ) 3225 3226 main.step( "Add 5 to then get a in-memory counter on each node" ) 3227 iCounters = [] 3228 threads = [] 3229 addedIValues = [] 3230 for i in range( main.numCtrls ): 3231 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3232 name="counterIncrement-" + str( i ), 3233 args=[ iCounterName ], 3234 kwargs={ "delta": 5, "inMemory": True } ) 3235 iCounterValue += 5 3236 addedIValues.append( iCounterValue ) 3237 threads.append( t ) 3238 t.start() 3239 3240 for t in threads: 3241 t.join() 3242 iCounters.append( t.result ) 3243 # Check that counter incremented numController times 3244 iCounterResults = True 3245 for i in addedIValues: 3246 tmpResult = i in iCounters 3247 iCounterResults = iCounterResults and tmpResult 3248 if not tmpResult: 3249 main.log.error( str( i ) + " is not in in-memory " 3250 "counter incremented results" ) 3251 utilities.assert_equals( expect=True, 3252 actual=pCounterResults, 3253 onpass="In-memory counter incremented", 3254 onfail="Error incrementing in-memory" + 3255 " counter" ) 3256 3257 main.step( "Get then add 5 to a in-memory counter on each node" ) 3258 iCounters = [] 3259 threads = [] 3260 addedIValues = [] 3261 for i in range( main.numCtrls ): 3262 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3263 name="counterIncrement-" + str( i ), 3264 args=[ iCounterName ], 3265 kwargs={ "delta": 5, "inMemory": True } ) 3266 addedIValues.append( iCounterValue ) 3267 iCounterValue += 5 3268 threads.append( t ) 3269 t.start() 3270 3271 for t in threads: 3272 t.join() 3273 iCounters.append( t.result ) 3274 # Check that counter incremented numController times 3275 iCounterResults = True 3276 for i in addedIValues: 3277 tmpResult = i in iCounters 3278 iCounterResults = iCounterResults and tmpResult 3279 if not tmpResult: 3280 main.log.error( str( i ) + " is not in in-memory " 3281 "counter incremented results" ) 3282 utilities.assert_equals( expect=True, 3283 actual=iCounterResults, 3284 onpass="In-memory counter incremented", 3285 onfail="Error incrementing in-memory" + 3286 " counter" ) 3287 3288 main.step( "Counters we added have the correct values" ) 3289 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 3290 utilities.assert_equals( expect=main.TRUE, 3291 actual=incrementCheck, 3292 onpass="Added counters are correct", 3293 onfail="Added counters are incorrect" ) 3294 3295 main.step( "Check counters are consistant across nodes" ) 3296 onosCounters = [] 3297 threads = [] 3298 for i in range( main.numCtrls ): 3299 t = main.Thread( target=main.CLIs[i].counters, 3300 name="counters-" + str( i ) ) 3301 threads.append( t ) 3302 t.start() 3303 for t in threads: 3304 t.join() 3305 onosCounters.append( t.result ) 3306 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ] 3307 if all( tmp ): 3308 main.log.info( "Counters are consistent across all nodes" ) 3309 consistentCounterResults = main.TRUE 3310 else: 3311 main.log.error( "Counters are not consistent across all nodes" ) 3312 consistentCounterResults = main.FALSE 3313 utilities.assert_equals( expect=main.TRUE, 3314 actual=consistentCounterResults, 3315 onpass="ONOS counters are consistent " + 3316 "across nodes", 3317 onfail="ONOS Counters are inconsistent " + 3318 "across nodes" ) 3319 3320 main.step( "Counters we added have the correct values" ) 3321 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 3322 incrementCheck = incrementCheck and \ 3323 main.Counters.counterCheck( iCounterName, iCounterValue ) 3324 utilities.assert_equals( expect=main.TRUE, 3325 actual=incrementCheck, 3326 onpass="Added counters are correct", 3327 onfail="Added counters are incorrect" ) 3328 3329 # DISTRIBUTED SETS 3330 main.step( "Distributed Set get" ) 3331 size = len( onosSet ) 3332 getResponses = [] 3333 threads = [] 3334 for i in range( main.numCtrls ): 3335 t = main.Thread( target=main.CLIs[i].setTestGet, 3336 name="setTestGet-" + str( i ), 3337 args=[ onosSetName ] ) 3338 threads.append( t ) 3339 t.start() 3340 for t in threads: 3341 t.join() 3342 getResponses.append( t.result ) 3343 3344 getResults = main.TRUE 3345 for i in range( main.numCtrls ): 3346 if isinstance( getResponses[ i ], list): 3347 current = set( getResponses[ i ] ) 3348 if len( current ) == len( getResponses[ i ] ): 3349 # no repeats 3350 if onosSet != current: 3351 main.log.error( "ONOS" + str( i + 1 ) + 3352 " has incorrect view" + 3353 " of set " + onosSetName + ":\n" + 3354 str( getResponses[ i ] ) ) 3355 main.log.debug( "Expected: " + str( onosSet ) ) 3356 main.log.debug( "Actual: " + str( current ) ) 3357 getResults = main.FALSE 3358 else: 3359 # error, set is not a set 3360 main.log.error( "ONOS" + str( i + 1 ) + 3361 " has repeat elements in" + 3362 " set " + onosSetName + ":\n" + 3363 str( getResponses[ i ] ) ) 3364 getResults = main.FALSE 3365 elif getResponses[ i ] == main.ERROR: 3366 getResults = main.FALSE 3367 utilities.assert_equals( expect=main.TRUE, 3368 actual=getResults, 3369 onpass="Set elements are correct", 3370 onfail="Set elements are incorrect" ) 3371 3372 main.step( "Distributed Set size" ) 3373 sizeResponses = [] 3374 threads = [] 3375 for i in range( main.numCtrls ): 3376 t = main.Thread( target=main.CLIs[i].setTestSize, 3377 name="setTestSize-" + str( i ), 3378 args=[ onosSetName ] ) 3379 threads.append( t ) 3380 t.start() 3381 for t in threads: 3382 t.join() 3383 sizeResponses.append( t.result ) 3384 3385 sizeResults = main.TRUE 3386 for i in range( main.numCtrls ): 3387 if size != sizeResponses[ i ]: 3388 sizeResults = main.FALSE 3389 main.log.error( "ONOS" + str( i + 1 ) + 3390 " expected a size of " + str( size ) + 3391 " for set " + onosSetName + 3392 " but got " + str( sizeResponses[ i ] ) ) 3393 utilities.assert_equals( expect=main.TRUE, 3394 actual=sizeResults, 3395 onpass="Set sizes are correct", 3396 onfail="Set sizes are incorrect" ) 3397 3398 main.step( "Distributed Set add()" ) 3399 onosSet.add( addValue ) 3400 addResponses = [] 3401 threads = [] 3402 for i in range( main.numCtrls ): 3403 t = main.Thread( target=main.CLIs[i].setTestAdd, 3404 name="setTestAdd-" + str( i ), 3405 args=[ onosSetName, addValue ] ) 3406 threads.append( t ) 3407 t.start() 3408 for t in threads: 3409 t.join() 3410 addResponses.append( t.result ) 3411 3412 # main.TRUE = successfully changed the set 3413 # main.FALSE = action resulted in no change in set 3414 # main.ERROR - Some error in executing the function 3415 addResults = main.TRUE 3416 for i in range( main.numCtrls ): 3417 if addResponses[ i ] == main.TRUE: 3418 # All is well 3419 pass 3420 elif addResponses[ i ] == main.FALSE: 3421 # Already in set, probably fine 3422 pass 3423 elif addResponses[ i ] == main.ERROR: 3424 # Error in execution 3425 addResults = main.FALSE 3426 else: 3427 # unexpected result 3428 addResults = main.FALSE 3429 if addResults != main.TRUE: 3430 main.log.error( "Error executing set add" ) 3431 3432 # Check if set is still correct 3433 size = len( onosSet ) 3434 getResponses = [] 3435 threads = [] 3436 for i in range( main.numCtrls ): 3437 t = main.Thread( target=main.CLIs[i].setTestGet, 3438 name="setTestGet-" + str( i ), 3439 args=[ onosSetName ] ) 3440 threads.append( t ) 3441 t.start() 3442 for t in threads: 3443 t.join() 3444 getResponses.append( t.result ) 3445 getResults = main.TRUE 3446 for i in range( main.numCtrls ): 3447 if isinstance( getResponses[ i ], list): 3448 current = set( getResponses[ i ] ) 3449 if len( current ) == len( getResponses[ i ] ): 3450 # no repeats 3451 if onosSet != current: 3452 main.log.error( "ONOS" + str( i + 1 ) + 3453 " has incorrect view" + 3454 " of set " + onosSetName + ":\n" + 3455 str( getResponses[ i ] ) ) 3456 main.log.debug( "Expected: " + str( onosSet ) ) 3457 main.log.debug( "Actual: " + str( current ) ) 3458 getResults = main.FALSE 3459 else: 3460 # error, set is not a set 3461 main.log.error( "ONOS" + str( i + 1 ) + 3462 " has repeat elements in" + 3463 " set " + onosSetName + ":\n" + 3464 str( getResponses[ i ] ) ) 3465 getResults = main.FALSE 3466 elif getResponses[ i ] == main.ERROR: 3467 getResults = main.FALSE 3468 sizeResponses = [] 3469 threads = [] 3470 for i in range( main.numCtrls ): 3471 t = main.Thread( target=main.CLIs[i].setTestSize, 3472 name="setTestSize-" + str( i ), 3473 args=[ onosSetName ] ) 3474 threads.append( t ) 3475 t.start() 3476 for t in threads: 3477 t.join() 3478 sizeResponses.append( t.result ) 3479 sizeResults = main.TRUE 3480 for i in range( main.numCtrls ): 3481 if size != sizeResponses[ i ]: 3482 sizeResults = main.FALSE 3483 main.log.error( "ONOS" + str( i + 1 ) + 3484 " expected a size of " + str( size ) + 3485 " for set " + onosSetName + 3486 " but got " + str( sizeResponses[ i ] ) ) 3487 addResults = addResults and getResults and sizeResults 3488 utilities.assert_equals( expect=main.TRUE, 3489 actual=addResults, 3490 onpass="Set add correct", 3491 onfail="Set add was incorrect" ) 3492 3493 main.step( "Distributed Set addAll()" ) 3494 onosSet.update( addAllValue.split() ) 3495 addResponses = [] 3496 threads = [] 3497 for i in range( main.numCtrls ): 3498 t = main.Thread( target=main.CLIs[i].setTestAdd, 3499 name="setTestAddAll-" + str( i ), 3500 args=[ onosSetName, addAllValue ] ) 3501 threads.append( t ) 3502 t.start() 3503 for t in threads: 3504 t.join() 3505 addResponses.append( t.result ) 3506 3507 # main.TRUE = successfully changed the set 3508 # main.FALSE = action resulted in no change in set 3509 # main.ERROR - Some error in executing the function 3510 addAllResults = main.TRUE 3511 for i in range( main.numCtrls ): 3512 if addResponses[ i ] == main.TRUE: 3513 # All is well 3514 pass 3515 elif addResponses[ i ] == main.FALSE: 3516 # Already in set, probably fine 3517 pass 3518 elif addResponses[ i ] == main.ERROR: 3519 # Error in execution 3520 addAllResults = main.FALSE 3521 else: 3522 # unexpected result 3523 addAllResults = main.FALSE 3524 if addAllResults != main.TRUE: 3525 main.log.error( "Error executing set addAll" ) 3526 3527 # Check if set is still correct 3528 size = len( onosSet ) 3529 getResponses = [] 3530 threads = [] 3531 for i in range( main.numCtrls ): 3532 t = main.Thread( target=main.CLIs[i].setTestGet, 3533 name="setTestGet-" + str( i ), 3534 args=[ onosSetName ] ) 3535 threads.append( t ) 3536 t.start() 3537 for t in threads: 3538 t.join() 3539 getResponses.append( t.result ) 3540 getResults = main.TRUE 3541 for i in range( main.numCtrls ): 3542 if isinstance( getResponses[ i ], list): 3543 current = set( getResponses[ i ] ) 3544 if len( current ) == len( getResponses[ i ] ): 3545 # no repeats 3546 if onosSet != current: 3547 main.log.error( "ONOS" + str( i + 1 ) + 3548 " has incorrect view" + 3549 " of set " + onosSetName + ":\n" + 3550 str( getResponses[ i ] ) ) 3551 main.log.debug( "Expected: " + str( onosSet ) ) 3552 main.log.debug( "Actual: " + str( current ) ) 3553 getResults = main.FALSE 3554 else: 3555 # error, set is not a set 3556 main.log.error( "ONOS" + str( i + 1 ) + 3557 " has repeat elements in" + 3558 " set " + onosSetName + ":\n" + 3559 str( getResponses[ i ] ) ) 3560 getResults = main.FALSE 3561 elif getResponses[ i ] == main.ERROR: 3562 getResults = main.FALSE 3563 sizeResponses = [] 3564 threads = [] 3565 for i in range( main.numCtrls ): 3566 t = main.Thread( target=main.CLIs[i].setTestSize, 3567 name="setTestSize-" + str( i ), 3568 args=[ onosSetName ] ) 3569 threads.append( t ) 3570 t.start() 3571 for t in threads: 3572 t.join() 3573 sizeResponses.append( t.result ) 3574 sizeResults = main.TRUE 3575 for i in range( main.numCtrls ): 3576 if size != sizeResponses[ i ]: 3577 sizeResults = main.FALSE 3578 main.log.error( "ONOS" + str( i + 1 ) + 3579 " expected a size of " + str( size ) + 3580 " for set " + onosSetName + 3581 " but got " + str( sizeResponses[ i ] ) ) 3582 addAllResults = addAllResults and getResults and sizeResults 3583 utilities.assert_equals( expect=main.TRUE, 3584 actual=addAllResults, 3585 onpass="Set addAll correct", 3586 onfail="Set addAll was incorrect" ) 3587 3588 main.step( "Distributed Set contains()" ) 3589 containsResponses = [] 3590 threads = [] 3591 for i in range( main.numCtrls ): 3592 t = main.Thread( target=main.CLIs[i].setTestGet, 3593 name="setContains-" + str( i ), 3594 args=[ onosSetName ], 3595 kwargs={ "values": addValue } ) 3596 threads.append( t ) 3597 t.start() 3598 for t in threads: 3599 t.join() 3600 # NOTE: This is the tuple 3601 containsResponses.append( t.result ) 3602 3603 containsResults = main.TRUE 3604 for i in range( main.numCtrls ): 3605 if containsResponses[ i ] == main.ERROR: 3606 containsResults = main.FALSE 3607 else: 3608 containsResults = containsResults and\ 3609 containsResponses[ i ][ 1 ] 3610 utilities.assert_equals( expect=main.TRUE, 3611 actual=containsResults, 3612 onpass="Set contains is functional", 3613 onfail="Set contains failed" ) 3614 3615 main.step( "Distributed Set containsAll()" ) 3616 containsAllResponses = [] 3617 threads = [] 3618 for i in range( main.numCtrls ): 3619 t = main.Thread( target=main.CLIs[i].setTestGet, 3620 name="setContainsAll-" + str( i ), 3621 args=[ onosSetName ], 3622 kwargs={ "values": addAllValue } ) 3623 threads.append( t ) 3624 t.start() 3625 for t in threads: 3626 t.join() 3627 # NOTE: This is the tuple 3628 containsAllResponses.append( t.result ) 3629 3630 containsAllResults = main.TRUE 3631 for i in range( main.numCtrls ): 3632 if containsResponses[ i ] == main.ERROR: 3633 containsResults = main.FALSE 3634 else: 3635 containsResults = containsResults and\ 3636 containsResponses[ i ][ 1 ] 3637 utilities.assert_equals( expect=main.TRUE, 3638 actual=containsAllResults, 3639 onpass="Set containsAll is functional", 3640 onfail="Set containsAll failed" ) 3641 3642 main.step( "Distributed Set remove()" ) 3643 onosSet.remove( addValue ) 3644 removeResponses = [] 3645 threads = [] 3646 for i in range( main.numCtrls ): 3647 t = main.Thread( target=main.CLIs[i].setTestRemove, 3648 name="setTestRemove-" + str( i ), 3649 args=[ onosSetName, addValue ] ) 3650 threads.append( t ) 3651 t.start() 3652 for t in threads: 3653 t.join() 3654 removeResponses.append( t.result ) 3655 3656 # main.TRUE = successfully changed the set 3657 # main.FALSE = action resulted in no change in set 3658 # main.ERROR - Some error in executing the function 3659 removeResults = main.TRUE 3660 for i in range( main.numCtrls ): 3661 if removeResponses[ i ] == main.TRUE: 3662 # All is well 3663 pass 3664 elif removeResponses[ i ] == main.FALSE: 3665 # not in set, probably fine 3666 pass 3667 elif removeResponses[ i ] == main.ERROR: 3668 # Error in execution 3669 removeResults = main.FALSE 3670 else: 3671 # unexpected result 3672 removeResults = main.FALSE 3673 if removeResults != main.TRUE: 3674 main.log.error( "Error executing set remove" ) 3675 3676 # Check if set is still correct 3677 size = len( onosSet ) 3678 getResponses = [] 3679 threads = [] 3680 for i in range( main.numCtrls ): 3681 t = main.Thread( target=main.CLIs[i].setTestGet, 3682 name="setTestGet-" + str( i ), 3683 args=[ onosSetName ] ) 3684 threads.append( t ) 3685 t.start() 3686 for t in threads: 3687 t.join() 3688 getResponses.append( t.result ) 3689 getResults = main.TRUE 3690 for i in range( main.numCtrls ): 3691 if isinstance( getResponses[ i ], list): 3692 current = set( getResponses[ i ] ) 3693 if len( current ) == len( getResponses[ i ] ): 3694 # no repeats 3695 if onosSet != current: 3696 main.log.error( "ONOS" + str( i + 1 ) + 3697 " has incorrect view" + 3698 " of set " + onosSetName + ":\n" + 3699 str( getResponses[ i ] ) ) 3700 main.log.debug( "Expected: " + str( onosSet ) ) 3701 main.log.debug( "Actual: " + str( current ) ) 3702 getResults = main.FALSE 3703 else: 3704 # error, set is not a set 3705 main.log.error( "ONOS" + str( i + 1 ) + 3706 " has repeat elements in" + 3707 " set " + onosSetName + ":\n" + 3708 str( getResponses[ i ] ) ) 3709 getResults = main.FALSE 3710 elif getResponses[ i ] == main.ERROR: 3711 getResults = main.FALSE 3712 sizeResponses = [] 3713 threads = [] 3714 for i in range( main.numCtrls ): 3715 t = main.Thread( target=main.CLIs[i].setTestSize, 3716 name="setTestSize-" + str( i ), 3717 args=[ onosSetName ] ) 3718 threads.append( t ) 3719 t.start() 3720 for t in threads: 3721 t.join() 3722 sizeResponses.append( t.result ) 3723 sizeResults = main.TRUE 3724 for i in range( main.numCtrls ): 3725 if size != sizeResponses[ i ]: 3726 sizeResults = main.FALSE 3727 main.log.error( "ONOS" + str( i + 1 ) + 3728 " expected a size of " + str( size ) + 3729 " for set " + onosSetName + 3730 " but got " + str( sizeResponses[ i ] ) ) 3731 removeResults = removeResults and getResults and sizeResults 3732 utilities.assert_equals( expect=main.TRUE, 3733 actual=removeResults, 3734 onpass="Set remove correct", 3735 onfail="Set remove was incorrect" ) 3736 3737 main.step( "Distributed Set removeAll()" ) 3738 onosSet.difference_update( addAllValue.split() ) 3739 removeAllResponses = [] 3740 threads = [] 3741 try: 3742 for i in range( main.numCtrls ): 3743 t = main.Thread( target=main.CLIs[i].setTestRemove, 3744 name="setTestRemoveAll-" + str( i ), 3745 args=[ onosSetName, addAllValue ] ) 3746 threads.append( t ) 3747 t.start() 3748 for t in threads: 3749 t.join() 3750 removeAllResponses.append( t.result ) 3751 except Exception, e: 3752 main.log.exception(e) 3753 3754 # main.TRUE = successfully changed the set 3755 # main.FALSE = action resulted in no change in set 3756 # main.ERROR - Some error in executing the function 3757 removeAllResults = main.TRUE 3758 for i in range( main.numCtrls ): 3759 if removeAllResponses[ i ] == main.TRUE: 3760 # All is well 3761 pass 3762 elif removeAllResponses[ i ] == main.FALSE: 3763 # not in set, probably fine 3764 pass 3765 elif removeAllResponses[ i ] == main.ERROR: 3766 # Error in execution 3767 removeAllResults = main.FALSE 3768 else: 3769 # unexpected result 3770 removeAllResults = main.FALSE 3771 if removeAllResults != main.TRUE: 3772 main.log.error( "Error executing set removeAll" ) 3773 3774 # Check if set is still correct 3775 size = len( onosSet ) 3776 getResponses = [] 3777 threads = [] 3778 for i in range( main.numCtrls ): 3779 t = main.Thread( target=main.CLIs[i].setTestGet, 3780 name="setTestGet-" + str( i ), 3781 args=[ onosSetName ] ) 3782 threads.append( t ) 3783 t.start() 3784 for t in threads: 3785 t.join() 3786 getResponses.append( t.result ) 3787 getResults = main.TRUE 3788 for i in range( main.numCtrls ): 3789 if isinstance( getResponses[ i ], list): 3790 current = set( getResponses[ i ] ) 3791 if len( current ) == len( getResponses[ i ] ): 3792 # no repeats 3793 if onosSet != current: 3794 main.log.error( "ONOS" + str( i + 1 ) + 3795 " has incorrect view" + 3796 " of set " + onosSetName + ":\n" + 3797 str( getResponses[ i ] ) ) 3798 main.log.debug( "Expected: " + str( onosSet ) ) 3799 main.log.debug( "Actual: " + str( current ) ) 3800 getResults = main.FALSE 3801 else: 3802 # error, set is not a set 3803 main.log.error( "ONOS" + str( i + 1 ) + 3804 " has repeat elements in" + 3805 " set " + onosSetName + ":\n" + 3806 str( getResponses[ i ] ) ) 3807 getResults = main.FALSE 3808 elif getResponses[ i ] == main.ERROR: 3809 getResults = main.FALSE 3810 sizeResponses = [] 3811 threads = [] 3812 for i in range( main.numCtrls ): 3813 t = main.Thread( target=main.CLIs[i].setTestSize, 3814 name="setTestSize-" + str( i ), 3815 args=[ onosSetName ] ) 3816 threads.append( t ) 3817 t.start() 3818 for t in threads: 3819 t.join() 3820 sizeResponses.append( t.result ) 3821 sizeResults = main.TRUE 3822 for i in range( main.numCtrls ): 3823 if size != sizeResponses[ i ]: 3824 sizeResults = main.FALSE 3825 main.log.error( "ONOS" + str( i + 1 ) + 3826 " expected a size of " + str( size ) + 3827 " for set " + onosSetName + 3828 " but got " + str( sizeResponses[ i ] ) ) 3829 removeAllResults = removeAllResults and getResults and sizeResults 3830 utilities.assert_equals( expect=main.TRUE, 3831 actual=removeAllResults, 3832 onpass="Set removeAll correct", 3833 onfail="Set removeAll was incorrect" ) 3834 3835 main.step( "Distributed Set addAll()" ) 3836 onosSet.update( addAllValue.split() ) 3837 addResponses = [] 3838 threads = [] 3839 for i in range( main.numCtrls ): 3840 t = main.Thread( target=main.CLIs[i].setTestAdd, 3841 name="setTestAddAll-" + str( i ), 3842 args=[ onosSetName, addAllValue ] ) 3843 threads.append( t ) 3844 t.start() 3845 for t in threads: 3846 t.join() 3847 addResponses.append( t.result ) 3848 3849 # main.TRUE = successfully changed the set 3850 # main.FALSE = action resulted in no change in set 3851 # main.ERROR - Some error in executing the function 3852 addAllResults = main.TRUE 3853 for i in range( main.numCtrls ): 3854 if addResponses[ i ] == main.TRUE: 3855 # All is well 3856 pass 3857 elif addResponses[ i ] == main.FALSE: 3858 # Already in set, probably fine 3859 pass 3860 elif addResponses[ i ] == main.ERROR: 3861 # Error in execution 3862 addAllResults = main.FALSE 3863 else: 3864 # unexpected result 3865 addAllResults = main.FALSE 3866 if addAllResults != main.TRUE: 3867 main.log.error( "Error executing set addAll" ) 3868 3869 # Check if set is still correct 3870 size = len( onosSet ) 3871 getResponses = [] 3872 threads = [] 3873 for i in range( main.numCtrls ): 3874 t = main.Thread( target=main.CLIs[i].setTestGet, 3875 name="setTestGet-" + str( i ), 3876 args=[ onosSetName ] ) 3877 threads.append( t ) 3878 t.start() 3879 for t in threads: 3880 t.join() 3881 getResponses.append( t.result ) 3882 getResults = main.TRUE 3883 for i in range( main.numCtrls ): 3884 if isinstance( getResponses[ i ], list): 3885 current = set( getResponses[ i ] ) 3886 if len( current ) == len( getResponses[ i ] ): 3887 # no repeats 3888 if onosSet != current: 3889 main.log.error( "ONOS" + str( i + 1 ) + 3890 " has incorrect view" + 3891 " of set " + onosSetName + ":\n" + 3892 str( getResponses[ i ] ) ) 3893 main.log.debug( "Expected: " + str( onosSet ) ) 3894 main.log.debug( "Actual: " + str( current ) ) 3895 getResults = main.FALSE 3896 else: 3897 # error, set is not a set 3898 main.log.error( "ONOS" + str( i + 1 ) + 3899 " has repeat elements in" + 3900 " set " + onosSetName + ":\n" + 3901 str( getResponses[ i ] ) ) 3902 getResults = main.FALSE 3903 elif getResponses[ i ] == main.ERROR: 3904 getResults = main.FALSE 3905 sizeResponses = [] 3906 threads = [] 3907 for i in range( main.numCtrls ): 3908 t = main.Thread( target=main.CLIs[i].setTestSize, 3909 name="setTestSize-" + str( i ), 3910 args=[ onosSetName ] ) 3911 threads.append( t ) 3912 t.start() 3913 for t in threads: 3914 t.join() 3915 sizeResponses.append( t.result ) 3916 sizeResults = main.TRUE 3917 for i in range( main.numCtrls ): 3918 if size != sizeResponses[ i ]: 3919 sizeResults = main.FALSE 3920 main.log.error( "ONOS" + str( i + 1 ) + 3921 " expected a size of " + str( size ) + 3922 " for set " + onosSetName + 3923 " but got " + str( sizeResponses[ i ] ) ) 3924 addAllResults = addAllResults and getResults and sizeResults 3925 utilities.assert_equals( expect=main.TRUE, 3926 actual=addAllResults, 3927 onpass="Set addAll correct", 3928 onfail="Set addAll was incorrect" ) 3929 3930 main.step( "Distributed Set clear()" ) 3931 onosSet.clear() 3932 clearResponses = [] 3933 threads = [] 3934 for i in range( main.numCtrls ): 3935 t = main.Thread( target=main.CLIs[i].setTestRemove, 3936 name="setTestClear-" + str( i ), 3937 args=[ onosSetName, " "], # Values doesn't matter 3938 kwargs={ "clear": True } ) 3939 threads.append( t ) 3940 t.start() 3941 for t in threads: 3942 t.join() 3943 clearResponses.append( t.result ) 3944 3945 # main.TRUE = successfully changed the set 3946 # main.FALSE = action resulted in no change in set 3947 # main.ERROR - Some error in executing the function 3948 clearResults = main.TRUE 3949 for i in range( main.numCtrls ): 3950 if clearResponses[ i ] == main.TRUE: 3951 # All is well 3952 pass 3953 elif clearResponses[ i ] == main.FALSE: 3954 # Nothing set, probably fine 3955 pass 3956 elif clearResponses[ i ] == main.ERROR: 3957 # Error in execution 3958 clearResults = main.FALSE 3959 else: 3960 # unexpected result 3961 clearResults = main.FALSE 3962 if clearResults != main.TRUE: 3963 main.log.error( "Error executing set clear" ) 3964 3965 # Check if set is still correct 3966 size = len( onosSet ) 3967 getResponses = [] 3968 threads = [] 3969 for i in range( main.numCtrls ): 3970 t = main.Thread( target=main.CLIs[i].setTestGet, 3971 name="setTestGet-" + str( i ), 3972 args=[ onosSetName ] ) 3973 threads.append( t ) 3974 t.start() 3975 for t in threads: 3976 t.join() 3977 getResponses.append( t.result ) 3978 getResults = main.TRUE 3979 for i in range( main.numCtrls ): 3980 if isinstance( getResponses[ i ], list): 3981 current = set( getResponses[ i ] ) 3982 if len( current ) == len( getResponses[ i ] ): 3983 # no repeats 3984 if onosSet != current: 3985 main.log.error( "ONOS" + str( i + 1 ) + 3986 " has incorrect view" + 3987 " of set " + onosSetName + ":\n" + 3988 str( getResponses[ i ] ) ) 3989 main.log.debug( "Expected: " + str( onosSet ) ) 3990 main.log.debug( "Actual: " + str( current ) ) 3991 getResults = main.FALSE 3992 else: 3993 # error, set is not a set 3994 main.log.error( "ONOS" + str( i + 1 ) + 3995 " has repeat elements in" + 3996 " set " + onosSetName + ":\n" + 3997 str( getResponses[ i ] ) ) 3998 getResults = main.FALSE 3999 elif getResponses[ i ] == main.ERROR: 4000 getResults = main.FALSE 4001 sizeResponses = [] 4002 threads = [] 4003 for i in range( main.numCtrls ): 4004 t = main.Thread( target=main.CLIs[i].setTestSize, 4005 name="setTestSize-" + str( i ), 4006 args=[ onosSetName ] ) 4007 threads.append( t ) 4008 t.start() 4009 for t in threads: 4010 t.join() 4011 sizeResponses.append( t.result ) 4012 sizeResults = main.TRUE 4013 for i in range( main.numCtrls ): 4014 if size != sizeResponses[ i ]: 4015 sizeResults = main.FALSE 4016 main.log.error( "ONOS" + str( i + 1 ) + 4017 " expected a size of " + str( size ) + 4018 " for set " + onosSetName + 4019 " but got " + str( sizeResponses[ i ] ) ) 4020 clearResults = clearResults and getResults and sizeResults 4021 utilities.assert_equals( expect=main.TRUE, 4022 actual=clearResults, 4023 onpass="Set clear correct", 4024 onfail="Set clear was incorrect" ) 4025 4026 main.step( "Distributed Set addAll()" ) 4027 onosSet.update( addAllValue.split() ) 4028 addResponses = [] 4029 threads = [] 4030 for i in range( main.numCtrls ): 4031 t = main.Thread( target=main.CLIs[i].setTestAdd, 4032 name="setTestAddAll-" + str( i ), 4033 args=[ onosSetName, addAllValue ] ) 4034 threads.append( t ) 4035 t.start() 4036 for t in threads: 4037 t.join() 4038 addResponses.append( t.result ) 4039 4040 # main.TRUE = successfully changed the set 4041 # main.FALSE = action resulted in no change in set 4042 # main.ERROR - Some error in executing the function 4043 addAllResults = main.TRUE 4044 for i in range( main.numCtrls ): 4045 if addResponses[ i ] == main.TRUE: 4046 # All is well 4047 pass 4048 elif addResponses[ i ] == main.FALSE: 4049 # Already in set, probably fine 4050 pass 4051 elif addResponses[ i ] == main.ERROR: 4052 # Error in execution 4053 addAllResults = main.FALSE 4054 else: 4055 # unexpected result 4056 addAllResults = main.FALSE 4057 if addAllResults != main.TRUE: 4058 main.log.error( "Error executing set addAll" ) 4059 4060 # Check if set is still correct 4061 size = len( onosSet ) 4062 getResponses = [] 4063 threads = [] 4064 for i in range( main.numCtrls ): 4065 t = main.Thread( target=main.CLIs[i].setTestGet, 4066 name="setTestGet-" + str( i ), 4067 args=[ onosSetName ] ) 4068 threads.append( t ) 4069 t.start() 4070 for t in threads: 4071 t.join() 4072 getResponses.append( t.result ) 4073 getResults = main.TRUE 4074 for i in range( main.numCtrls ): 4075 if isinstance( getResponses[ i ], list): 4076 current = set( getResponses[ i ] ) 4077 if len( current ) == len( getResponses[ i ] ): 4078 # no repeats 4079 if onosSet != current: 4080 main.log.error( "ONOS" + str( i + 1 ) + 4081 " has incorrect view" + 4082 " of set " + onosSetName + ":\n" + 4083 str( getResponses[ i ] ) ) 4084 main.log.debug( "Expected: " + str( onosSet ) ) 4085 main.log.debug( "Actual: " + str( current ) ) 4086 getResults = main.FALSE 4087 else: 4088 # error, set is not a set 4089 main.log.error( "ONOS" + str( i + 1 ) + 4090 " has repeat elements in" + 4091 " set " + onosSetName + ":\n" + 4092 str( getResponses[ i ] ) ) 4093 getResults = main.FALSE 4094 elif getResponses[ i ] == main.ERROR: 4095 getResults = main.FALSE 4096 sizeResponses = [] 4097 threads = [] 4098 for i in range( main.numCtrls ): 4099 t = main.Thread( target=main.CLIs[i].setTestSize, 4100 name="setTestSize-" + str( i ), 4101 args=[ onosSetName ] ) 4102 threads.append( t ) 4103 t.start() 4104 for t in threads: 4105 t.join() 4106 sizeResponses.append( t.result ) 4107 sizeResults = main.TRUE 4108 for i in range( main.numCtrls ): 4109 if size != sizeResponses[ i ]: 4110 sizeResults = main.FALSE 4111 main.log.error( "ONOS" + str( i + 1 ) + 4112 " expected a size of " + str( size ) + 4113 " for set " + onosSetName + 4114 " but got " + str( sizeResponses[ i ] ) ) 4115 addAllResults = addAllResults and getResults and sizeResults 4116 utilities.assert_equals( expect=main.TRUE, 4117 actual=addAllResults, 4118 onpass="Set addAll correct", 4119 onfail="Set addAll was incorrect" ) 4120 4121 main.step( "Distributed Set retain()" ) 4122 onosSet.intersection_update( retainValue.split() ) 4123 retainResponses = [] 4124 threads = [] 4125 for i in range( main.numCtrls ): 4126 t = main.Thread( target=main.CLIs[i].setTestRemove, 4127 name="setTestRetain-" + str( i ), 4128 args=[ onosSetName, retainValue ], 4129 kwargs={ "retain": True } ) 4130 threads.append( t ) 4131 t.start() 4132 for t in threads: 4133 t.join() 4134 retainResponses.append( t.result ) 4135 4136 # main.TRUE = successfully changed the set 4137 # main.FALSE = action resulted in no change in set 4138 # main.ERROR - Some error in executing the function 4139 retainResults = main.TRUE 4140 for i in range( main.numCtrls ): 4141 if retainResponses[ i ] == main.TRUE: 4142 # All is well 4143 pass 4144 elif retainResponses[ i ] == main.FALSE: 4145 # Already in set, probably fine 4146 pass 4147 elif retainResponses[ i ] == main.ERROR: 4148 # Error in execution 4149 retainResults = main.FALSE 4150 else: 4151 # unexpected result 4152 retainResults = main.FALSE 4153 if retainResults != main.TRUE: 4154 main.log.error( "Error executing set retain" ) 4155 4156 # Check if set is still correct 4157 size = len( onosSet ) 4158 getResponses = [] 4159 threads = [] 4160 for i in range( main.numCtrls ): 4161 t = main.Thread( target=main.CLIs[i].setTestGet, 4162 name="setTestGet-" + str( i ), 4163 args=[ onosSetName ] ) 4164 threads.append( t ) 4165 t.start() 4166 for t in threads: 4167 t.join() 4168 getResponses.append( t.result ) 4169 getResults = main.TRUE 4170 for i in range( main.numCtrls ): 4171 if isinstance( getResponses[ i ], list): 4172 current = set( getResponses[ i ] ) 4173 if len( current ) == len( getResponses[ i ] ): 4174 # no repeats 4175 if onosSet != current: 4176 main.log.error( "ONOS" + str( i + 1 ) + 4177 " has incorrect view" + 4178 " of set " + onosSetName + ":\n" + 4179 str( getResponses[ i ] ) ) 4180 main.log.debug( "Expected: " + str( onosSet ) ) 4181 main.log.debug( "Actual: " + str( current ) ) 4182 getResults = main.FALSE 4183 else: 4184 # error, set is not a set 4185 main.log.error( "ONOS" + str( i + 1 ) + 4186 " has repeat elements in" + 4187 " set " + onosSetName + ":\n" + 4188 str( getResponses[ i ] ) ) 4189 getResults = main.FALSE 4190 elif getResponses[ i ] == main.ERROR: 4191 getResults = main.FALSE 4192 sizeResponses = [] 4193 threads = [] 4194 for i in range( main.numCtrls ): 4195 t = main.Thread( target=main.CLIs[i].setTestSize, 4196 name="setTestSize-" + str( i ), 4197 args=[ onosSetName ] ) 4198 threads.append( t ) 4199 t.start() 4200 for t in threads: 4201 t.join() 4202 sizeResponses.append( t.result ) 4203 sizeResults = main.TRUE 4204 for i in range( main.numCtrls ): 4205 if size != sizeResponses[ i ]: 4206 sizeResults = main.FALSE 4207 main.log.error( "ONOS" + str( i + 1 ) + 4208 " expected a size of " + 4209 str( size ) + " for set " + onosSetName + 4210 " but got " + str( sizeResponses[ i ] ) ) 4211 retainResults = retainResults and getResults and sizeResults 4212 utilities.assert_equals( expect=main.TRUE, 4213 actual=retainResults, 4214 onpass="Set retain correct", 4215 onfail="Set retain was incorrect" )
4216