Package TestON :: Package tests :: Package HAclusterRestart :: Module HAclusterRestart
[hide private]
[frames] | no frames]

Source Code for Module TestON.tests.HAclusterRestart.HAclusterRestart

   1  """ 
   2  Description: This test is to determine if ONOS can handle 
   3      all of it's nodes restarting 
   4   
   5  List of test cases: 
   6  CASE1: Compile ONOS and push it to the test machines 
   7  CASE2: Assign devices to controllers 
   8  CASE21: Assign mastership to controllers 
   9  CASE3: Assign intents 
  10  CASE4: Ping across added host intents 
  11  CASE5: Reading state of ONOS 
  12  CASE6: The Failure case. 
  13  CASE7: Check state after control plane failure 
  14  CASE8: Compare topo 
  15  CASE9: Link s3-s28 down 
  16  CASE10: Link s3-s28 up 
  17  CASE11: Switch down 
  18  CASE12: Switch up 
  19  CASE13: Clean up 
  20  CASE14: start election app on all onos nodes 
  21  CASE15: Check that Leadership Election is still functional 
  22  CASE16: Install Distributed Primitives app 
  23  CASE17: Check for basic functionality with distributed primitives 
  24  """ 
  25   
  26   
27 -class HAclusterRestart:
28
29 - def __init__( self ):
30 self.default = ''
31
32 - def CASE1( self, main ):
33 """ 34 CASE1 is to compile ONOS and push it to the test machines 35 36 Startup sequence: 37 cell <name> 38 onos-verify-cell 39 NOTE: temporary - onos-remove-raft-logs 40 onos-uninstall 41 start mininet 42 git pull 43 mvn clean install 44 onos-package 45 onos-install -f 46 onos-wait-for-start 47 start cli sessions 48 start tcpdump 49 """ 50 import imp 51 main.log.info( "ONOS HA test: Restart all ONOS nodes - " + 52 "initialization" ) 53 main.case( "Setting up test environment" ) 54 main.caseExplanation = "Setup the test environment including " +\ 55 "installing ONOS, starting Mininet and ONOS" +\ 56 "cli sessions." 57 # TODO: save all the timers and output them for plotting 58 59 # load some variables from the params file 60 PULLCODE = False 61 if main.params[ 'Git' ] == 'True': 62 PULLCODE = True 63 gitBranch = main.params[ 'branch' ] 64 cellName = main.params[ 'ENV' ][ 'cellName' ] 65 66 main.numCtrls = int( main.params[ 'num_controllers' ] ) 67 if main.ONOSbench.maxNodes: 68 if main.ONOSbench.maxNodes < main.numCtrls: 69 main.numCtrls = int( main.ONOSbench.maxNodes ) 70 # set global variables 71 global ONOS1Port 72 global ONOS2Port 73 global ONOS3Port 74 global ONOS4Port 75 global ONOS5Port 76 global ONOS6Port 77 global ONOS7Port 78 # These are for csv plotting in jenkins 79 global labels 80 global data 81 labels = [] 82 data = [] 83 84 # FIXME: just get controller port from params? 85 # TODO: do we really need all these? 86 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ] 87 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ] 88 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ] 89 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ] 90 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ] 91 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ] 92 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ] 93 94 try: 95 fileName = "Counters" 96 path = main.params[ 'imports' ][ 'path' ] 97 main.Counters = imp.load_source( fileName, 98 path + fileName + ".py" ) 99 except Exception as e: 100 main.log.exception( e ) 101 main.cleanup() 102 main.exit() 103 104 main.CLIs = [] 105 main.nodes = [] 106 ipList = [] 107 for i in range( 1, main.numCtrls + 1 ): 108 try: 109 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) ) 110 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) ) 111 ipList.append( main.nodes[ -1 ].ip_address ) 112 except AttributeError: 113 break 114 115 main.step( "Create cell file" ) 116 cellAppString = main.params[ 'ENV' ][ 'appString' ] 117 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName, 118 main.Mininet1.ip_address, 119 cellAppString, ipList ) 120 main.step( "Applying cell variable to environment" ) 121 cellResult = main.ONOSbench.setCell( cellName ) 122 verifyResult = main.ONOSbench.verifyCell() 123 124 # FIXME:this is short term fix 125 main.log.info( "Removing raft logs" ) 126 main.ONOSbench.onosRemoveRaftLogs() 127 128 main.log.info( "Uninstalling ONOS" ) 129 for node in main.nodes: 130 main.ONOSbench.onosUninstall( node.ip_address ) 131 132 # Make sure ONOS is DEAD 133 main.log.info( "Killing any ONOS processes" ) 134 killResults = main.TRUE 135 for node in main.nodes: 136 killed = main.ONOSbench.onosKill( node.ip_address ) 137 killResults = killResults and killed 138 139 cleanInstallResult = main.TRUE 140 gitPullResult = main.TRUE 141 142 main.step( "Starting Mininet" ) 143 # scp topo file to mininet 144 # TODO: move to params? 145 topoName = "obelisk.py" 146 filePath = main.ONOSbench.home + "/tools/test/topos/" 147 main.ONOSbench.copyMininetFile( topoName, filePath, 148 main.Mininet1.user_name, 149 main.Mininet1.ip_address ) 150 mnResult = main.Mininet1.startNet( ) 151 utilities.assert_equals( expect=main.TRUE, actual=mnResult, 152 onpass="Mininet Started", 153 onfail="Error starting Mininet" ) 154 155 main.step( "Git checkout and pull " + gitBranch ) 156 if PULLCODE: 157 main.ONOSbench.gitCheckout( gitBranch ) 158 gitPullResult = main.ONOSbench.gitPull() 159 # values of 1 or 3 are good 160 utilities.assert_lesser( expect=0, actual=gitPullResult, 161 onpass="Git pull successful", 162 onfail="Git pull failed" ) 163 main.ONOSbench.getVersion( report=True ) 164 165 main.step( "Using mvn clean install" ) 166 cleanInstallResult = main.TRUE 167 if PULLCODE and gitPullResult == main.TRUE: 168 cleanInstallResult = main.ONOSbench.cleanInstall() 169 else: 170 main.log.warn( "Did not pull new code so skipping mvn " + 171 "clean install" ) 172 utilities.assert_equals( expect=main.TRUE, 173 actual=cleanInstallResult, 174 onpass="MCI successful", 175 onfail="MCI failed" ) 176 # GRAPHS 177 # NOTE: important params here: 178 # job = name of Jenkins job 179 # Plot Name = Plot-HA, only can be used if multiple plots 180 # index = The number of the graph under plot name 181 job = "HAclusterRestart" 182 plotName = "Plot-HA" 183 graphs = '<ac:structured-macro ac:name="html">\n' 184 graphs += '<ac:plain-text-body><![CDATA[\n' 185 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\ 186 '/plot/' + plotName + '/getPlot?index=0' +\ 187 '&width=500&height=300"' +\ 188 'noborder="0" width="500" height="300" scrolling="yes" ' +\ 189 'seamless="seamless"></iframe>\n' 190 graphs += ']]></ac:plain-text-body>\n' 191 graphs += '</ac:structured-macro>\n' 192 main.log.wiki(graphs) 193 194 main.step( "Creating ONOS package" ) 195 packageResult = main.ONOSbench.onosPackage() 196 utilities.assert_equals( expect=main.TRUE, actual=packageResult, 197 onpass="ONOS package successful", 198 onfail="ONOS package failed" ) 199 200 main.step( "Installing ONOS package" ) 201 onosInstallResult = main.TRUE 202 for node in main.nodes: 203 tmpResult = main.ONOSbench.onosInstall( options="-f", 204 node=node.ip_address ) 205 onosInstallResult = onosInstallResult and tmpResult 206 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult, 207 onpass="ONOS install successful", 208 onfail="ONOS install failed" ) 209 210 main.step( "Checking if ONOS is up yet" ) 211 for i in range( 2 ): 212 onosIsupResult = main.TRUE 213 for node in main.nodes: 214 started = main.ONOSbench.isup( node.ip_address ) 215 if not started: 216 main.log.error( node.name + " didn't start!" ) 217 main.ONOSbench.onosStop( node.ip_address ) 218 main.ONOSbench.onosStart( node.ip_address ) 219 onosIsupResult = onosIsupResult and started 220 if onosIsupResult == main.TRUE: 221 break 222 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult, 223 onpass="ONOS startup successful", 224 onfail="ONOS startup failed" ) 225 226 main.log.step( "Starting ONOS CLI sessions" ) 227 cliResults = main.TRUE 228 threads = [] 229 for i in range( main.numCtrls ): 230 t = main.Thread( target=main.CLIs[i].startOnosCli, 231 name="startOnosCli-" + str( i ), 232 args=[main.nodes[i].ip_address] ) 233 threads.append( t ) 234 t.start() 235 236 for t in threads: 237 t.join() 238 cliResults = cliResults and t.result 239 utilities.assert_equals( expect=main.TRUE, actual=cliResults, 240 onpass="ONOS cli startup successful", 241 onfail="ONOS cli startup failed" ) 242 243 if main.params[ 'tcpdump' ].lower() == "true": 244 main.step( "Start Packet Capture MN" ) 245 main.Mininet2.startTcpdump( 246 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST ) 247 + "-MN.pcap", 248 intf=main.params[ 'MNtcpdump' ][ 'intf' ], 249 port=main.params[ 'MNtcpdump' ][ 'port' ] ) 250 251 main.step( "App Ids check" ) 252 appCheck = main.TRUE 253 threads = [] 254 for i in range( main.numCtrls ): 255 t = main.Thread( target=main.CLIs[i].appToIDCheck, 256 name="appToIDCheck-" + str( i ), 257 args=[] ) 258 threads.append( t ) 259 t.start() 260 261 for t in threads: 262 t.join() 263 appCheck = appCheck and t.result 264 if appCheck != main.TRUE: 265 main.log.warn( main.CLIs[0].apps() ) 266 main.log.warn( main.CLIs[0].appIDs() ) 267 utilities.assert_equals( expect=main.TRUE, actual=appCheck, 268 onpass="App Ids seem to be correct", 269 onfail="Something is wrong with app Ids" ) 270 271 if cliResults == main.FALSE: 272 main.log.error( "Failed to start ONOS, stopping test" ) 273 main.cleanup() 274 main.exit()
275
276 - def CASE2( self, main ):
277 """ 278 Assign devices to controllers 279 """ 280 import re 281 assert main.numCtrls, "main.numCtrls not defined" 282 assert main, "main not defined" 283 assert utilities.assert_equals, "utilities.assert_equals not defined" 284 assert main.CLIs, "main.CLIs not defined" 285 assert main.nodes, "main.nodes not defined" 286 assert ONOS1Port, "ONOS1Port not defined" 287 assert ONOS2Port, "ONOS2Port not defined" 288 assert ONOS3Port, "ONOS3Port not defined" 289 assert ONOS4Port, "ONOS4Port not defined" 290 assert ONOS5Port, "ONOS5Port not defined" 291 assert ONOS6Port, "ONOS6Port not defined" 292 assert ONOS7Port, "ONOS7Port not defined" 293 294 main.case( "Assigning devices to controllers" ) 295 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\ 296 "and check that an ONOS node becomes the " +\ 297 "master of the device." 298 main.step( "Assign switches to controllers" ) 299 300 ipList = [] 301 for i in range( main.numCtrls ): 302 ipList.append( main.nodes[ i ].ip_address ) 303 swList = [] 304 for i in range( 1, 29 ): 305 swList.append( "s" + str( i ) ) 306 main.Mininet1.assignSwController( sw=swList, ip=ipList ) 307 308 mastershipCheck = main.TRUE 309 for i in range( 1, 29 ): 310 response = main.Mininet1.getSwController( "s" + str( i ) ) 311 try: 312 main.log.info( str( response ) ) 313 except Exception: 314 main.log.info( repr( response ) ) 315 for node in main.nodes: 316 if re.search( "tcp:" + node.ip_address, response ): 317 mastershipCheck = mastershipCheck and main.TRUE 318 else: 319 main.log.error( "Error, node " + node.ip_address + " is " + 320 "not in the list of controllers s" + 321 str( i ) + " is connecting to." ) 322 mastershipCheck = main.FALSE 323 utilities.assert_equals( 324 expect=main.TRUE, 325 actual=mastershipCheck, 326 onpass="Switch mastership assigned correctly", 327 onfail="Switches not assigned correctly to controllers" )
328
329 - def CASE21( self, main ):
330 """ 331 Assign mastership to controllers 332 """ 333 import time 334 assert main.numCtrls, "main.numCtrls not defined" 335 assert main, "main not defined" 336 assert utilities.assert_equals, "utilities.assert_equals not defined" 337 assert main.CLIs, "main.CLIs not defined" 338 assert main.nodes, "main.nodes not defined" 339 assert ONOS1Port, "ONOS1Port not defined" 340 assert ONOS2Port, "ONOS2Port not defined" 341 assert ONOS3Port, "ONOS3Port not defined" 342 assert ONOS4Port, "ONOS4Port not defined" 343 assert ONOS5Port, "ONOS5Port not defined" 344 assert ONOS6Port, "ONOS6Port not defined" 345 assert ONOS7Port, "ONOS7Port not defined" 346 347 main.case( "Assigning Controller roles for switches" ) 348 main.caseExplanation = "Check that ONOS is connected to each " +\ 349 "device. Then manually assign" +\ 350 " mastership to specific ONOS nodes using" +\ 351 " 'device-role'" 352 main.step( "Assign mastership of switches to specific controllers" ) 353 # Manually assign mastership to the controller we want 354 roleCall = main.TRUE 355 356 ipList = [ ] 357 deviceList = [] 358 try: 359 # Assign mastership to specific controllers. This assignment was 360 # determined for a 7 node cluser, but will work with any sized 361 # cluster 362 for i in range( 1, 29 ): # switches 1 through 28 363 # set up correct variables: 364 if i == 1: 365 c = 0 366 ip = main.nodes[ c ].ip_address # ONOS1 367 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' ) 368 elif i == 2: 369 c = 1 % main.numCtrls 370 ip = main.nodes[ c ].ip_address # ONOS2 371 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' ) 372 elif i == 3: 373 c = 1 % main.numCtrls 374 ip = main.nodes[ c ].ip_address # ONOS2 375 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' ) 376 elif i == 4: 377 c = 3 % main.numCtrls 378 ip = main.nodes[ c ].ip_address # ONOS4 379 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' ) 380 elif i == 5: 381 c = 2 % main.numCtrls 382 ip = main.nodes[ c ].ip_address # ONOS3 383 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' ) 384 elif i == 6: 385 c = 2 % main.numCtrls 386 ip = main.nodes[ c ].ip_address # ONOS3 387 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' ) 388 elif i == 7: 389 c = 5 % main.numCtrls 390 ip = main.nodes[ c ].ip_address # ONOS6 391 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' ) 392 elif i >= 8 and i <= 17: 393 c = 4 % main.numCtrls 394 ip = main.nodes[ c ].ip_address # ONOS5 395 dpid = '3' + str( i ).zfill( 3 ) 396 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' ) 397 elif i >= 18 and i <= 27: 398 c = 6 % main.numCtrls 399 ip = main.nodes[ c ].ip_address # ONOS7 400 dpid = '6' + str( i ).zfill( 3 ) 401 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' ) 402 elif i == 28: 403 c = 0 404 ip = main.nodes[ c ].ip_address # ONOS1 405 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' ) 406 else: 407 main.log.error( "You didn't write an else statement for " + 408 "switch s" + str( i ) ) 409 roleCall = main.FALSE 410 # Assign switch 411 assert deviceId, "No device id for s" + str( i ) + " in ONOS" 412 # TODO: make this controller dynamic 413 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId, 414 ip ) 415 ipList.append( ip ) 416 deviceList.append( deviceId ) 417 except ( AttributeError, AssertionError ): 418 main.log.exception( "Something is wrong with ONOS device view" ) 419 main.log.info( main.ONOScli1.devices() ) 420 utilities.assert_equals( 421 expect=main.TRUE, 422 actual=roleCall, 423 onpass="Re-assigned switch mastership to designated controller", 424 onfail="Something wrong with deviceRole calls" ) 425 426 main.step( "Check mastership was correctly assigned" ) 427 roleCheck = main.TRUE 428 # NOTE: This is due to the fact that device mastership change is not 429 # atomic and is actually a multi step process 430 time.sleep( 5 ) 431 for i in range( len( ipList ) ): 432 ip = ipList[i] 433 deviceId = deviceList[i] 434 # Check assignment 435 master = main.ONOScli1.getRole( deviceId ).get( 'master' ) 436 if ip in master: 437 roleCheck = roleCheck and main.TRUE 438 else: 439 roleCheck = roleCheck and main.FALSE 440 main.log.error( "Error, controller " + ip + " is not" + 441 " master " + "of device " + 442 str( deviceId ) + ". Master is " + 443 repr( master ) + "." ) 444 utilities.assert_equals( 445 expect=main.TRUE, 446 actual=roleCheck, 447 onpass="Switches were successfully reassigned to designated " + 448 "controller", 449 onfail="Switches were not successfully reassigned" )
450
451 - def CASE3( self, main ):
452 """ 453 Assign intents 454 """ 455 import time 456 import json 457 assert main.numCtrls, "main.numCtrls not defined" 458 assert main, "main not defined" 459 assert utilities.assert_equals, "utilities.assert_equals not defined" 460 assert main.CLIs, "main.CLIs not defined" 461 assert main.nodes, "main.nodes not defined" 462 try: 463 labels 464 except NameError: 465 main.log.error( "labels not defined, setting to []" ) 466 labels = [] 467 try: 468 data 469 except NameError: 470 main.log.error( "data not defined, setting to []" ) 471 data = [] 472 # NOTE: we must reinstall intents until we have a persistant intent 473 # datastore! 474 main.case( "Adding host Intents" ) 475 main.caseExplanation = "Discover hosts by using pingall then " +\ 476 "assign predetermined host-to-host intents." +\ 477 " After installation, check that the intent" +\ 478 " is distributed to all nodes and the state" +\ 479 " is INSTALLED" 480 481 # install onos-app-fwd 482 main.step( "Install reactive forwarding app" ) 483 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" ) 484 utilities.assert_equals( expect=main.TRUE, actual=installResults, 485 onpass="Install fwd successful", 486 onfail="Install fwd failed" ) 487 488 main.step( "Check app ids" ) 489 appCheck = main.TRUE 490 threads = [] 491 for i in range( main.numCtrls ): 492 t = main.Thread( target=main.CLIs[i].appToIDCheck, 493 name="appToIDCheck-" + str( i ), 494 args=[] ) 495 threads.append( t ) 496 t.start() 497 498 for t in threads: 499 t.join() 500 appCheck = appCheck and t.result 501 if appCheck != main.TRUE: 502 main.log.warn( main.CLIs[0].apps() ) 503 main.log.warn( main.CLIs[0].appIDs() ) 504 utilities.assert_equals( expect=main.TRUE, actual=appCheck, 505 onpass="App Ids seem to be correct", 506 onfail="Something is wrong with app Ids" ) 507 508 main.step( "Discovering Hosts( Via pingall for now )" ) 509 # FIXME: Once we have a host discovery mechanism, use that instead 510 # REACTIVE FWD test 511 pingResult = main.FALSE 512 for i in range(2): # Retry if pingall fails first time 513 time1 = time.time() 514 pingResult = main.Mininet1.pingall() 515 if i == 0: 516 utilities.assert_equals( 517 expect=main.TRUE, 518 actual=pingResult, 519 onpass="Reactive Pingall test passed", 520 onfail="Reactive Pingall failed, " + 521 "one or more ping pairs failed" ) 522 time2 = time.time() 523 main.log.info( "Time for pingall: %2f seconds" % 524 ( time2 - time1 ) ) 525 # timeout for fwd flows 526 time.sleep( 11 ) 527 # uninstall onos-app-fwd 528 main.step( "Uninstall reactive forwarding app" ) 529 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" ) 530 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult, 531 onpass="Uninstall fwd successful", 532 onfail="Uninstall fwd failed" ) 533 534 main.step( "Check app ids" ) 535 threads = [] 536 appCheck2 = main.TRUE 537 for i in range( main.numCtrls ): 538 t = main.Thread( target=main.CLIs[i].appToIDCheck, 539 name="appToIDCheck-" + str( i ), 540 args=[] ) 541 threads.append( t ) 542 t.start() 543 544 for t in threads: 545 t.join() 546 appCheck2 = appCheck2 and t.result 547 if appCheck2 != main.TRUE: 548 main.log.warn( main.CLIs[0].apps() ) 549 main.log.warn( main.CLIs[0].appIDs() ) 550 utilities.assert_equals( expect=main.TRUE, actual=appCheck2, 551 onpass="App Ids seem to be correct", 552 onfail="Something is wrong with app Ids" ) 553 554 main.step( "Add host intents via cli" ) 555 intentIds = [] 556 # TODO: move the host numbers to params 557 # Maybe look at all the paths we ping? 558 intentAddResult = True 559 hostResult = main.TRUE 560 for i in range( 8, 18 ): 561 main.log.info( "Adding host intent between h" + str( i ) + 562 " and h" + str( i + 10 ) ) 563 host1 = "00:00:00:00:00:" + \ 564 str( hex( i )[ 2: ] ).zfill( 2 ).upper() 565 host2 = "00:00:00:00:00:" + \ 566 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper() 567 # NOTE: getHost can return None 568 host1Dict = main.ONOScli1.getHost( host1 ) 569 host2Dict = main.ONOScli1.getHost( host2 ) 570 host1Id = None 571 host2Id = None 572 if host1Dict and host2Dict: 573 host1Id = host1Dict.get( 'id', None ) 574 host2Id = host2Dict.get( 'id', None ) 575 if host1Id and host2Id: 576 nodeNum = ( i % main.numCtrls ) 577 tmpId = main.CLIs[ nodeNum ].addHostIntent( host1Id, host2Id ) 578 if tmpId: 579 main.log.info( "Added intent with id: " + tmpId ) 580 intentIds.append( tmpId ) 581 else: 582 main.log.error( "addHostIntent returned: " + 583 repr( tmpId ) ) 584 else: 585 main.log.error( "Error, getHost() failed for h" + str( i ) + 586 " and/or h" + str( i + 10 ) ) 587 hosts = main.CLIs[ 0 ].hosts() 588 main.log.warn( "Hosts output: " ) 589 try: 590 main.log.warn( json.dumps( json.loads( hosts ), 591 sort_keys=True, 592 indent=4, 593 separators=( ',', ': ' ) ) ) 594 except ( ValueError, TypeError ): 595 main.log.warn( repr( hosts ) ) 596 hostResult = main.FALSE 597 utilities.assert_equals( expect=main.TRUE, actual=hostResult, 598 onpass="Found a host id for each host", 599 onfail="Error looking up host ids" ) 600 601 intentStart = time.time() 602 onosIds = main.ONOScli1.getAllIntentsId() 603 main.log.info( "Submitted intents: " + str( intentIds ) ) 604 main.log.info( "Intents in ONOS: " + str( onosIds ) ) 605 for intent in intentIds: 606 if intent in onosIds: 607 pass # intent submitted is in onos 608 else: 609 intentAddResult = False 610 if intentAddResult: 611 intentStop = time.time() 612 else: 613 intentStop = None 614 # Print the intent states 615 intents = main.ONOScli1.intents() 616 intentStates = [] 617 installedCheck = True 618 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 619 count = 0 620 try: 621 for intent in json.loads( intents ): 622 state = intent.get( 'state', None ) 623 if "INSTALLED" not in state: 624 installedCheck = False 625 intentId = intent.get( 'id', None ) 626 intentStates.append( ( intentId, state ) ) 627 except ( ValueError, TypeError ): 628 main.log.exception( "Error parsing intents" ) 629 # add submitted intents not in the store 630 tmplist = [ i for i, s in intentStates ] 631 missingIntents = False 632 for i in intentIds: 633 if i not in tmplist: 634 intentStates.append( ( i, " - " ) ) 635 missingIntents = True 636 intentStates.sort() 637 for i, s in intentStates: 638 count += 1 639 main.log.info( "%-6s%-15s%-15s" % 640 ( str( count ), str( i ), str( s ) ) ) 641 leaders = main.ONOScli1.leaders() 642 try: 643 missing = False 644 if leaders: 645 parsedLeaders = json.loads( leaders ) 646 main.log.warn( json.dumps( parsedLeaders, 647 sort_keys=True, 648 indent=4, 649 separators=( ',', ': ' ) ) ) 650 # check for all intent partitions 651 topics = [] 652 for i in range( 14 ): 653 topics.append( "intent-partition-" + str( i ) ) 654 main.log.debug( topics ) 655 ONOStopics = [ j['topic'] for j in parsedLeaders ] 656 for topic in topics: 657 if topic not in ONOStopics: 658 main.log.error( "Error: " + topic + 659 " not in leaders" ) 660 missing = True 661 else: 662 main.log.error( "leaders() returned None" ) 663 except ( ValueError, TypeError ): 664 main.log.exception( "Error parsing leaders" ) 665 main.log.error( repr( leaders ) ) 666 # Check all nodes 667 if missing: 668 for node in main.CLIs: 669 response = node.leaders( jsonFormat=False) 670 main.log.warn( str( node.name ) + " leaders output: \n" + 671 str( response ) ) 672 673 partitions = main.ONOScli1.partitions() 674 try: 675 if partitions : 676 parsedPartitions = json.loads( partitions ) 677 main.log.warn( json.dumps( parsedPartitions, 678 sort_keys=True, 679 indent=4, 680 separators=( ',', ': ' ) ) ) 681 # TODO check for a leader in all paritions 682 # TODO check for consistency among nodes 683 else: 684 main.log.error( "partitions() returned None" ) 685 except ( ValueError, TypeError ): 686 main.log.exception( "Error parsing partitions" ) 687 main.log.error( repr( partitions ) ) 688 pendingMap = main.ONOScli1.pendingMap() 689 try: 690 if pendingMap : 691 parsedPending = json.loads( pendingMap ) 692 main.log.warn( json.dumps( parsedPending, 693 sort_keys=True, 694 indent=4, 695 separators=( ',', ': ' ) ) ) 696 # TODO check something here? 697 else: 698 main.log.error( "pendingMap() returned None" ) 699 except ( ValueError, TypeError ): 700 main.log.exception( "Error parsing pending map" ) 701 main.log.error( repr( pendingMap ) ) 702 703 intentAddResult = bool( intentAddResult and not missingIntents and 704 installedCheck ) 705 if not intentAddResult: 706 main.log.error( "Error in pushing host intents to ONOS" ) 707 708 main.step( "Intent Anti-Entropy dispersion" ) 709 for i in range(100): 710 correct = True 711 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) ) 712 for cli in main.CLIs: 713 onosIds = [] 714 ids = cli.getAllIntentsId() 715 onosIds.append( ids ) 716 main.log.debug( "Intents in " + cli.name + ": " + 717 str( sorted( onosIds ) ) ) 718 if sorted( ids ) != sorted( intentIds ): 719 main.log.warn( "Set of intent IDs doesn't match" ) 720 correct = False 721 break 722 else: 723 intents = json.loads( cli.intents() ) 724 for intent in intents: 725 if intent[ 'state' ] != "INSTALLED": 726 main.log.warn( "Intent " + intent[ 'id' ] + 727 " is " + intent[ 'state' ] ) 728 correct = False 729 break 730 if correct: 731 break 732 else: 733 time.sleep(1) 734 if not intentStop: 735 intentStop = time.time() 736 global gossipTime 737 gossipTime = intentStop - intentStart 738 main.log.info( "It took about " + str( gossipTime ) + 739 " seconds for all intents to appear in each node" ) 740 append = False 741 title = "Gossip Intents" 742 count = 1 743 while append is False: 744 curTitle = title + str( count ) 745 if curTitle not in labels: 746 labels.append( curTitle ) 747 data.append( str( gossipTime ) ) 748 append = True 749 else: 750 count += 1 751 # FIXME: make this time configurable/calculate based off of number of 752 # nodes and gossip rounds 753 utilities.assert_greater_equals( 754 expect=40, actual=gossipTime, 755 onpass="ECM anti-entropy for intents worked within " + 756 "expected time", 757 onfail="Intent ECM anti-entropy took too long" ) 758 if gossipTime <= 40: 759 intentAddResult = True 760 761 if not intentAddResult or "key" in pendingMap: 762 import time 763 installedCheck = True 764 main.log.info( "Sleeping 60 seconds to see if intents are found" ) 765 time.sleep( 60 ) 766 onosIds = main.ONOScli1.getAllIntentsId() 767 main.log.info( "Submitted intents: " + str( intentIds ) ) 768 main.log.info( "Intents in ONOS: " + str( onosIds ) ) 769 # Print the intent states 770 intents = main.ONOScli1.intents() 771 intentStates = [] 772 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 773 count = 0 774 try: 775 for intent in json.loads( intents ): 776 # Iter through intents of a node 777 state = intent.get( 'state', None ) 778 if "INSTALLED" not in state: 779 installedCheck = False 780 intentId = intent.get( 'id', None ) 781 intentStates.append( ( intentId, state ) ) 782 except ( ValueError, TypeError ): 783 main.log.exception( "Error parsing intents" ) 784 # add submitted intents not in the store 785 tmplist = [ i for i, s in intentStates ] 786 for i in intentIds: 787 if i not in tmplist: 788 intentStates.append( ( i, " - " ) ) 789 intentStates.sort() 790 for i, s in intentStates: 791 count += 1 792 main.log.info( "%-6s%-15s%-15s" % 793 ( str( count ), str( i ), str( s ) ) ) 794 leaders = main.ONOScli1.leaders() 795 try: 796 missing = False 797 if leaders: 798 parsedLeaders = json.loads( leaders ) 799 main.log.warn( json.dumps( parsedLeaders, 800 sort_keys=True, 801 indent=4, 802 separators=( ',', ': ' ) ) ) 803 # check for all intent partitions 804 # check for election 805 topics = [] 806 for i in range( 14 ): 807 topics.append( "intent-partition-" + str( i ) ) 808 # FIXME: this should only be after we start the app 809 topics.append( "org.onosproject.election" ) 810 main.log.debug( topics ) 811 ONOStopics = [ j['topic'] for j in parsedLeaders ] 812 for topic in topics: 813 if topic not in ONOStopics: 814 main.log.error( "Error: " + topic + 815 " not in leaders" ) 816 missing = True 817 else: 818 main.log.error( "leaders() returned None" ) 819 except ( ValueError, TypeError ): 820 main.log.exception( "Error parsing leaders" ) 821 main.log.error( repr( leaders ) ) 822 # Check all nodes 823 if missing: 824 for node in main.CLIs: 825 response = node.leaders( jsonFormat=False) 826 main.log.warn( str( node.name ) + " leaders output: \n" + 827 str( response ) ) 828 829 partitions = main.ONOScli1.partitions() 830 try: 831 if partitions : 832 parsedPartitions = json.loads( partitions ) 833 main.log.warn( json.dumps( parsedPartitions, 834 sort_keys=True, 835 indent=4, 836 separators=( ',', ': ' ) ) ) 837 # TODO check for a leader in all paritions 838 # TODO check for consistency among nodes 839 else: 840 main.log.error( "partitions() returned None" ) 841 except ( ValueError, TypeError ): 842 main.log.exception( "Error parsing partitions" ) 843 main.log.error( repr( partitions ) ) 844 pendingMap = main.ONOScli1.pendingMap() 845 try: 846 if pendingMap : 847 parsedPending = json.loads( pendingMap ) 848 main.log.warn( json.dumps( parsedPending, 849 sort_keys=True, 850 indent=4, 851 separators=( ',', ': ' ) ) ) 852 # TODO check something here? 853 else: 854 main.log.error( "pendingMap() returned None" ) 855 except ( ValueError, TypeError ): 856 main.log.exception( "Error parsing pending map" ) 857 main.log.error( repr( pendingMap ) )
858
859 - def CASE4( self, main ):
860 """ 861 Ping across added host intents 862 """ 863 import json 864 import time 865 assert main.numCtrls, "main.numCtrls not defined" 866 assert main, "main not defined" 867 assert utilities.assert_equals, "utilities.assert_equals not defined" 868 assert main.CLIs, "main.CLIs not defined" 869 assert main.nodes, "main.nodes not defined" 870 main.case( "Verify connectivity by sendind traffic across Intents" ) 871 main.caseExplanation = "Ping across added host intents to check " +\ 872 "functionality and check the state of " +\ 873 "the intent" 874 main.step( "Ping across added host intents" ) 875 PingResult = main.TRUE 876 for i in range( 8, 18 ): 877 ping = main.Mininet1.pingHost( src="h" + str( i ), 878 target="h" + str( i + 10 ) ) 879 PingResult = PingResult and ping 880 if ping == main.FALSE: 881 main.log.warn( "Ping failed between h" + str( i ) + 882 " and h" + str( i + 10 ) ) 883 elif ping == main.TRUE: 884 main.log.info( "Ping test passed!" ) 885 # Don't set PingResult or you'd override failures 886 if PingResult == main.FALSE: 887 main.log.error( 888 "Intents have not been installed correctly, pings failed." ) 889 # TODO: pretty print 890 main.log.warn( "ONOS1 intents: " ) 891 try: 892 tmpIntents = main.ONOScli1.intents() 893 main.log.warn( json.dumps( json.loads( tmpIntents ), 894 sort_keys=True, 895 indent=4, 896 separators=( ',', ': ' ) ) ) 897 except ( ValueError, TypeError ): 898 main.log.warn( repr( tmpIntents ) ) 899 utilities.assert_equals( 900 expect=main.TRUE, 901 actual=PingResult, 902 onpass="Intents have been installed correctly and pings work", 903 onfail="Intents have not been installed correctly, pings failed." ) 904 905 main.step( "Check Intent state" ) 906 installedCheck = False 907 loopCount = 0 908 while not installedCheck and loopCount < 40: 909 installedCheck = True 910 # Print the intent states 911 intents = main.ONOScli1.intents() 912 intentStates = [] 913 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 914 count = 0 915 # Iter through intents of a node 916 try: 917 for intent in json.loads( intents ): 918 state = intent.get( 'state', None ) 919 if "INSTALLED" not in state: 920 installedCheck = False 921 intentId = intent.get( 'id', None ) 922 intentStates.append( ( intentId, state ) ) 923 except ( ValueError, TypeError ): 924 main.log.exception( "Error parsing intents." ) 925 # Print states 926 intentStates.sort() 927 for i, s in intentStates: 928 count += 1 929 main.log.info( "%-6s%-15s%-15s" % 930 ( str( count ), str( i ), str( s ) ) ) 931 if not installedCheck: 932 time.sleep( 1 ) 933 loopCount += 1 934 utilities.assert_equals( expect=True, actual=installedCheck, 935 onpass="Intents are all INSTALLED", 936 onfail="Intents are not all in " + 937 "INSTALLED state" ) 938 939 main.step( "Check leadership of topics" ) 940 leaders = main.ONOScli1.leaders() 941 topicCheck = main.TRUE 942 try: 943 if leaders: 944 parsedLeaders = json.loads( leaders ) 945 main.log.warn( json.dumps( parsedLeaders, 946 sort_keys=True, 947 indent=4, 948 separators=( ',', ': ' ) ) ) 949 # check for all intent partitions 950 # check for election 951 # TODO: Look at Devices as topics now that it uses this system 952 topics = [] 953 for i in range( 14 ): 954 topics.append( "intent-partition-" + str( i ) ) 955 # FIXME: this should only be after we start the app 956 # FIXME: topics.append( "org.onosproject.election" ) 957 # Print leaders output 958 main.log.debug( topics ) 959 ONOStopics = [ j['topic'] for j in parsedLeaders ] 960 for topic in topics: 961 if topic not in ONOStopics: 962 main.log.error( "Error: " + topic + 963 " not in leaders" ) 964 topicCheck = main.FALSE 965 else: 966 main.log.error( "leaders() returned None" ) 967 topicCheck = main.FALSE 968 except ( ValueError, TypeError ): 969 topicCheck = main.FALSE 970 main.log.exception( "Error parsing leaders" ) 971 main.log.error( repr( leaders ) ) 972 # TODO: Check for a leader of these topics 973 # Check all nodes 974 if topicCheck: 975 for node in main.CLIs: 976 response = node.leaders( jsonFormat=False) 977 main.log.warn( str( node.name ) + " leaders output: \n" + 978 str( response ) ) 979 980 utilities.assert_equals( expect=main.TRUE, actual=topicCheck, 981 onpass="intent Partitions is in leaders", 982 onfail="Some topics were lost " ) 983 # Print partitions 984 partitions = main.ONOScli1.partitions() 985 try: 986 if partitions : 987 parsedPartitions = json.loads( partitions ) 988 main.log.warn( json.dumps( parsedPartitions, 989 sort_keys=True, 990 indent=4, 991 separators=( ',', ': ' ) ) ) 992 # TODO check for a leader in all paritions 993 # TODO check for consistency among nodes 994 else: 995 main.log.error( "partitions() returned None" ) 996 except ( ValueError, TypeError ): 997 main.log.exception( "Error parsing partitions" ) 998 main.log.error( repr( partitions ) ) 999 # Print Pending Map 1000 pendingMap = main.ONOScli1.pendingMap() 1001 try: 1002 if pendingMap : 1003 parsedPending = json.loads( pendingMap ) 1004 main.log.warn( json.dumps( parsedPending, 1005 sort_keys=True, 1006 indent=4, 1007 separators=( ',', ': ' ) ) ) 1008 # TODO check something here? 1009 else: 1010 main.log.error( "pendingMap() returned None" ) 1011 except ( ValueError, TypeError ): 1012 main.log.exception( "Error parsing pending map" ) 1013 main.log.error( repr( pendingMap ) ) 1014 1015 if not installedCheck: 1016 main.log.info( "Waiting 60 seconds to see if the state of " + 1017 "intents change" ) 1018 time.sleep( 60 ) 1019 # Print the intent states 1020 intents = main.ONOScli1.intents() 1021 intentStates = [] 1022 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 1023 count = 0 1024 # Iter through intents of a node 1025 try: 1026 for intent in json.loads( intents ): 1027 state = intent.get( 'state', None ) 1028 if "INSTALLED" not in state: 1029 installedCheck = False 1030 intentId = intent.get( 'id', None ) 1031 intentStates.append( ( intentId, state ) ) 1032 except ( ValueError, TypeError ): 1033 main.log.exception( "Error parsing intents." ) 1034 intentStates.sort() 1035 for i, s in intentStates: 1036 count += 1 1037 main.log.info( "%-6s%-15s%-15s" % 1038 ( str( count ), str( i ), str( s ) ) ) 1039 leaders = main.ONOScli1.leaders() 1040 try: 1041 missing = False 1042 if leaders: 1043 parsedLeaders = json.loads( leaders ) 1044 main.log.warn( json.dumps( parsedLeaders, 1045 sort_keys=True, 1046 indent=4, 1047 separators=( ',', ': ' ) ) ) 1048 # check for all intent partitions 1049 # check for election 1050 topics = [] 1051 for i in range( 14 ): 1052 topics.append( "intent-partition-" + str( i ) ) 1053 # FIXME: this should only be after we start the app 1054 topics.append( "org.onosproject.election" ) 1055 main.log.debug( topics ) 1056 ONOStopics = [ j['topic'] for j in parsedLeaders ] 1057 for topic in topics: 1058 if topic not in ONOStopics: 1059 main.log.error( "Error: " + topic + 1060 " not in leaders" ) 1061 missing = True 1062 else: 1063 main.log.error( "leaders() returned None" ) 1064 except ( ValueError, TypeError ): 1065 main.log.exception( "Error parsing leaders" ) 1066 main.log.error( repr( leaders ) ) 1067 if missing: 1068 for node in main.CLIs: 1069 response = node.leaders( jsonFormat=False) 1070 main.log.warn( str( node.name ) + " leaders output: \n" + 1071 str( response ) ) 1072 1073 partitions = main.ONOScli1.partitions() 1074 try: 1075 if partitions : 1076 parsedPartitions = json.loads( partitions ) 1077 main.log.warn( json.dumps( parsedPartitions, 1078 sort_keys=True, 1079 indent=4, 1080 separators=( ',', ': ' ) ) ) 1081 # TODO check for a leader in all paritions 1082 # TODO check for consistency among nodes 1083 else: 1084 main.log.error( "partitions() returned None" ) 1085 except ( ValueError, TypeError ): 1086 main.log.exception( "Error parsing partitions" ) 1087 main.log.error( repr( partitions ) ) 1088 pendingMap = main.ONOScli1.pendingMap() 1089 try: 1090 if pendingMap : 1091 parsedPending = json.loads( pendingMap ) 1092 main.log.warn( json.dumps( parsedPending, 1093 sort_keys=True, 1094 indent=4, 1095 separators=( ',', ': ' ) ) ) 1096 # TODO check something here? 1097 else: 1098 main.log.error( "pendingMap() returned None" ) 1099 except ( ValueError, TypeError ): 1100 main.log.exception( "Error parsing pending map" ) 1101 main.log.error( repr( pendingMap ) ) 1102 # Print flowrules 1103 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) ) 1104 main.step( "Wait a minute then ping again" ) 1105 # the wait is above 1106 PingResult = main.TRUE 1107 for i in range( 8, 18 ): 1108 ping = main.Mininet1.pingHost( src="h" + str( i ), 1109 target="h" + str( i + 10 ) ) 1110 PingResult = PingResult and ping 1111 if ping == main.FALSE: 1112 main.log.warn( "Ping failed between h" + str( i ) + 1113 " and h" + str( i + 10 ) ) 1114 elif ping == main.TRUE: 1115 main.log.info( "Ping test passed!" ) 1116 # Don't set PingResult or you'd override failures 1117 if PingResult == main.FALSE: 1118 main.log.error( 1119 "Intents have not been installed correctly, pings failed." ) 1120 # TODO: pretty print 1121 main.log.warn( "ONOS1 intents: " ) 1122 try: 1123 tmpIntents = main.ONOScli1.intents() 1124 main.log.warn( json.dumps( json.loads( tmpIntents ), 1125 sort_keys=True, 1126 indent=4, 1127 separators=( ',', ': ' ) ) ) 1128 except ( ValueError, TypeError ): 1129 main.log.warn( repr( tmpIntents ) ) 1130 utilities.assert_equals( 1131 expect=main.TRUE, 1132 actual=PingResult, 1133 onpass="Intents have been installed correctly and pings work", 1134 onfail="Intents have not been installed correctly, pings failed." )
1135
1136 - def CASE5( self, main ):
1137 """ 1138 Reading state of ONOS 1139 """ 1140 import json 1141 import time 1142 assert main.numCtrls, "main.numCtrls not defined" 1143 assert main, "main not defined" 1144 assert utilities.assert_equals, "utilities.assert_equals not defined" 1145 assert main.CLIs, "main.CLIs not defined" 1146 assert main.nodes, "main.nodes not defined" 1147 1148 main.case( "Setting up and gathering data for current state" ) 1149 # The general idea for this test case is to pull the state of 1150 # ( intents,flows, topology,... ) from each ONOS node 1151 # We can then compare them with each other and also with past states 1152 1153 main.step( "Check that each switch has a master" ) 1154 global mastershipState 1155 mastershipState = '[]' 1156 1157 # Assert that each device has a master 1158 rolesNotNull = main.TRUE 1159 threads = [] 1160 for i in range( main.numCtrls ): 1161 t = main.Thread( target=main.CLIs[i].rolesNotNull, 1162 name="rolesNotNull-" + str( i ), 1163 args=[] ) 1164 threads.append( t ) 1165 t.start() 1166 1167 for t in threads: 1168 t.join() 1169 rolesNotNull = rolesNotNull and t.result 1170 utilities.assert_equals( 1171 expect=main.TRUE, 1172 actual=rolesNotNull, 1173 onpass="Each device has a master", 1174 onfail="Some devices don't have a master assigned" ) 1175 1176 main.step( "Get the Mastership of each switch from each controller" ) 1177 ONOSMastership = [] 1178 mastershipCheck = main.FALSE 1179 consistentMastership = True 1180 rolesResults = True 1181 threads = [] 1182 for i in range( main.numCtrls ): 1183 t = main.Thread( target=main.CLIs[i].roles, 1184 name="roles-" + str( i ), 1185 args=[] ) 1186 threads.append( t ) 1187 t.start() 1188 1189 for t in threads: 1190 t.join() 1191 ONOSMastership.append( t.result ) 1192 1193 for i in range( main.numCtrls ): 1194 if not ONOSMastership[i] or "Error" in ONOSMastership[i]: 1195 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1196 " roles" ) 1197 main.log.warn( 1198 "ONOS" + str( i + 1 ) + " mastership response: " + 1199 repr( ONOSMastership[i] ) ) 1200 rolesResults = False 1201 utilities.assert_equals( 1202 expect=True, 1203 actual=rolesResults, 1204 onpass="No error in reading roles output", 1205 onfail="Error in reading roles from ONOS" ) 1206 1207 main.step( "Check for consistency in roles from each controller" ) 1208 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ): 1209 main.log.info( 1210 "Switch roles are consistent across all ONOS nodes" ) 1211 else: 1212 consistentMastership = False 1213 utilities.assert_equals( 1214 expect=True, 1215 actual=consistentMastership, 1216 onpass="Switch roles are consistent across all ONOS nodes", 1217 onfail="ONOS nodes have different views of switch roles" ) 1218 1219 if rolesResults and not consistentMastership: 1220 for i in range( main.numCtrls ): 1221 try: 1222 main.log.warn( 1223 "ONOS" + str( i + 1 ) + " roles: ", 1224 json.dumps( 1225 json.loads( ONOSMastership[ i ] ), 1226 sort_keys=True, 1227 indent=4, 1228 separators=( ',', ': ' ) ) ) 1229 except ( ValueError, TypeError ): 1230 main.log.warn( repr( ONOSMastership[ i ] ) ) 1231 elif rolesResults and consistentMastership: 1232 mastershipCheck = main.TRUE 1233 mastershipState = ONOSMastership[ 0 ] 1234 1235 main.step( "Get the intents from each controller" ) 1236 global intentState 1237 intentState = [] 1238 ONOSIntents = [] 1239 intentCheck = main.FALSE 1240 consistentIntents = True 1241 intentsResults = True 1242 threads = [] 1243 for i in range( main.numCtrls ): 1244 t = main.Thread( target=main.CLIs[i].intents, 1245 name="intents-" + str( i ), 1246 args=[], 1247 kwargs={ 'jsonFormat': True } ) 1248 threads.append( t ) 1249 t.start() 1250 1251 for t in threads: 1252 t.join() 1253 ONOSIntents.append( t.result ) 1254 1255 for i in range( main.numCtrls ): 1256 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]: 1257 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1258 " intents" ) 1259 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " + 1260 repr( ONOSIntents[ i ] ) ) 1261 intentsResults = False 1262 utilities.assert_equals( 1263 expect=True, 1264 actual=intentsResults, 1265 onpass="No error in reading intents output", 1266 onfail="Error in reading intents from ONOS" ) 1267 1268 main.step( "Check for consistency in Intents from each controller" ) 1269 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ): 1270 main.log.info( "Intents are consistent across all ONOS " + 1271 "nodes" ) 1272 else: 1273 consistentIntents = False 1274 main.log.error( "Intents not consistent" ) 1275 utilities.assert_equals( 1276 expect=True, 1277 actual=consistentIntents, 1278 onpass="Intents are consistent across all ONOS nodes", 1279 onfail="ONOS nodes have different views of intents" ) 1280 1281 if intentsResults: 1282 # Try to make it easy to figure out what is happening 1283 # 1284 # Intent ONOS1 ONOS2 ... 1285 # 0x01 INSTALLED INSTALLING 1286 # ... ... ... 1287 # ... ... ... 1288 title = " Id" 1289 for n in range( main.numCtrls ): 1290 title += " " * 10 + "ONOS" + str( n + 1 ) 1291 main.log.warn( title ) 1292 # get all intent keys in the cluster 1293 keys = [] 1294 for nodeStr in ONOSIntents: 1295 node = json.loads( nodeStr ) 1296 for intent in node: 1297 keys.append( intent.get( 'id' ) ) 1298 keys = set( keys ) 1299 for key in keys: 1300 row = "%-13s" % key 1301 for nodeStr in ONOSIntents: 1302 node = json.loads( nodeStr ) 1303 for intent in node: 1304 if intent.get( 'id', "Error" ) == key: 1305 row += "%-15s" % intent.get( 'state' ) 1306 main.log.warn( row ) 1307 # End table view 1308 1309 if intentsResults and not consistentIntents: 1310 # print the json objects 1311 n = len(ONOSIntents) 1312 main.log.debug( "ONOS" + str( n ) + " intents: " ) 1313 main.log.debug( json.dumps( json.loads( ONOSIntents[ -1 ] ), 1314 sort_keys=True, 1315 indent=4, 1316 separators=( ',', ': ' ) ) ) 1317 for i in range( main.numCtrls ): 1318 if ONOSIntents[ i ] != ONOSIntents[ -1 ]: 1319 main.log.debug( "ONOS" + str( i + 1 ) + " intents: " ) 1320 main.log.debug( json.dumps( json.loads( ONOSIntents[i] ), 1321 sort_keys=True, 1322 indent=4, 1323 separators=( ',', ': ' ) ) ) 1324 else: 1325 main.log.debug( main.nodes[ i ].name + " intents match ONOS" + 1326 str( n ) + " intents" ) 1327 elif intentsResults and consistentIntents: 1328 intentCheck = main.TRUE 1329 intentState = ONOSIntents[ 0 ] 1330 1331 main.step( "Get the flows from each controller" ) 1332 global flowState 1333 flowState = [] 1334 ONOSFlows = [] 1335 ONOSFlowsJson = [] 1336 flowCheck = main.FALSE 1337 consistentFlows = True 1338 flowsResults = True 1339 threads = [] 1340 for i in range( main.numCtrls ): 1341 t = main.Thread( target=main.CLIs[i].flows, 1342 name="flows-" + str( i ), 1343 args=[], 1344 kwargs={ 'jsonFormat': True } ) 1345 threads.append( t ) 1346 t.start() 1347 1348 # NOTE: Flows command can take some time to run 1349 time.sleep(30) 1350 for t in threads: 1351 t.join() 1352 result = t.result 1353 ONOSFlows.append( result ) 1354 1355 for i in range( main.numCtrls ): 1356 num = str( i + 1 ) 1357 if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]: 1358 main.log.error( "Error in getting ONOS" + num + " flows" ) 1359 main.log.warn( "ONOS" + num + " flows response: " + 1360 repr( ONOSFlows[ i ] ) ) 1361 flowsResults = False 1362 ONOSFlowsJson.append( None ) 1363 else: 1364 try: 1365 ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) ) 1366 except ( ValueError, TypeError ): 1367 # FIXME: change this to log.error? 1368 main.log.exception( "Error in parsing ONOS" + num + 1369 " response as json." ) 1370 main.log.error( repr( ONOSFlows[ i ] ) ) 1371 ONOSFlowsJson.append( None ) 1372 flowsResults = False 1373 utilities.assert_equals( 1374 expect=True, 1375 actual=flowsResults, 1376 onpass="No error in reading flows output", 1377 onfail="Error in reading flows from ONOS" ) 1378 1379 main.step( "Check for consistency in Flows from each controller" ) 1380 tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ] 1381 if all( tmp ): 1382 main.log.info( "Flow count is consistent across all ONOS nodes" ) 1383 else: 1384 consistentFlows = False 1385 utilities.assert_equals( 1386 expect=True, 1387 actual=consistentFlows, 1388 onpass="The flow count is consistent across all ONOS nodes", 1389 onfail="ONOS nodes have different flow counts" ) 1390 1391 if flowsResults and not consistentFlows: 1392 for i in range( main.numCtrls ): 1393 try: 1394 main.log.warn( 1395 "ONOS" + str( i + 1 ) + " flows: " + 1396 json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True, 1397 indent=4, separators=( ',', ': ' ) ) ) 1398 except ( ValueError, TypeError ): 1399 main.log.warn( 1400 "ONOS" + str( i + 1 ) + " flows: " + 1401 repr( ONOSFlows[ i ] ) ) 1402 elif flowsResults and consistentFlows: 1403 flowCheck = main.TRUE 1404 flowState = ONOSFlows[ 0 ] 1405 1406 main.step( "Get the OF Table entries" ) 1407 global flows 1408 flows = [] 1409 for i in range( 1, 29 ): 1410 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) ) 1411 if flowCheck == main.FALSE: 1412 for table in flows: 1413 main.log.warn( table ) 1414 # TODO: Compare switch flow tables with ONOS flow tables 1415 1416 main.step( "Start continuous pings" ) 1417 main.Mininet2.pingLong( 1418 src=main.params[ 'PING' ][ 'source1' ], 1419 target=main.params[ 'PING' ][ 'target1' ], 1420 pingTime=500 ) 1421 main.Mininet2.pingLong( 1422 src=main.params[ 'PING' ][ 'source2' ], 1423 target=main.params[ 'PING' ][ 'target2' ], 1424 pingTime=500 ) 1425 main.Mininet2.pingLong( 1426 src=main.params[ 'PING' ][ 'source3' ], 1427 target=main.params[ 'PING' ][ 'target3' ], 1428 pingTime=500 ) 1429 main.Mininet2.pingLong( 1430 src=main.params[ 'PING' ][ 'source4' ], 1431 target=main.params[ 'PING' ][ 'target4' ], 1432 pingTime=500 ) 1433 main.Mininet2.pingLong( 1434 src=main.params[ 'PING' ][ 'source5' ], 1435 target=main.params[ 'PING' ][ 'target5' ], 1436 pingTime=500 ) 1437 main.Mininet2.pingLong( 1438 src=main.params[ 'PING' ][ 'source6' ], 1439 target=main.params[ 'PING' ][ 'target6' ], 1440 pingTime=500 ) 1441 main.Mininet2.pingLong( 1442 src=main.params[ 'PING' ][ 'source7' ], 1443 target=main.params[ 'PING' ][ 'target7' ], 1444 pingTime=500 ) 1445 main.Mininet2.pingLong( 1446 src=main.params[ 'PING' ][ 'source8' ], 1447 target=main.params[ 'PING' ][ 'target8' ], 1448 pingTime=500 ) 1449 main.Mininet2.pingLong( 1450 src=main.params[ 'PING' ][ 'source9' ], 1451 target=main.params[ 'PING' ][ 'target9' ], 1452 pingTime=500 ) 1453 main.Mininet2.pingLong( 1454 src=main.params[ 'PING' ][ 'source10' ], 1455 target=main.params[ 'PING' ][ 'target10' ], 1456 pingTime=500 ) 1457 1458 main.step( "Collecting topology information from ONOS" ) 1459 devices = [] 1460 threads = [] 1461 for i in range( main.numCtrls ): 1462 t = main.Thread( target=main.CLIs[i].devices, 1463 name="devices-" + str( i ), 1464 args=[ ] ) 1465 threads.append( t ) 1466 t.start() 1467 1468 for t in threads: 1469 t.join() 1470 devices.append( t.result ) 1471 hosts = [] 1472 threads = [] 1473 for i in range( main.numCtrls ): 1474 t = main.Thread( target=main.CLIs[i].hosts, 1475 name="hosts-" + str( i ), 1476 args=[ ] ) 1477 threads.append( t ) 1478 t.start() 1479 1480 for t in threads: 1481 t.join() 1482 try: 1483 hosts.append( json.loads( t.result ) ) 1484 except ( ValueError, TypeError ): 1485 # FIXME: better handling of this, print which node 1486 # Maybe use thread name? 1487 main.log.exception( "Error parsing json output of hosts" ) 1488 # FIXME: should this be an empty json object instead? 1489 hosts.append( None ) 1490 1491 ports = [] 1492 threads = [] 1493 for i in range( main.numCtrls ): 1494 t = main.Thread( target=main.CLIs[i].ports, 1495 name="ports-" + str( i ), 1496 args=[ ] ) 1497 threads.append( t ) 1498 t.start() 1499 1500 for t in threads: 1501 t.join() 1502 ports.append( t.result ) 1503 links = [] 1504 threads = [] 1505 for i in range( main.numCtrls ): 1506 t = main.Thread( target=main.CLIs[i].links, 1507 name="links-" + str( i ), 1508 args=[ ] ) 1509 threads.append( t ) 1510 t.start() 1511 1512 for t in threads: 1513 t.join() 1514 links.append( t.result ) 1515 clusters = [] 1516 threads = [] 1517 for i in range( main.numCtrls ): 1518 t = main.Thread( target=main.CLIs[i].clusters, 1519 name="clusters-" + str( i ), 1520 args=[ ] ) 1521 threads.append( t ) 1522 t.start() 1523 1524 for t in threads: 1525 t.join() 1526 clusters.append( t.result ) 1527 # Compare json objects for hosts and dataplane clusters 1528 1529 # hosts 1530 main.step( "Host view is consistent across ONOS nodes" ) 1531 consistentHostsResult = main.TRUE 1532 for controller in range( len( hosts ) ): 1533 controllerStr = str( controller + 1 ) 1534 if "Error" not in hosts[ controller ]: 1535 if hosts[ controller ] == hosts[ 0 ]: 1536 continue 1537 else: # hosts not consistent 1538 main.log.error( "hosts from ONOS" + 1539 controllerStr + 1540 " is inconsistent with ONOS1" ) 1541 main.log.warn( repr( hosts[ controller ] ) ) 1542 consistentHostsResult = main.FALSE 1543 1544 else: 1545 main.log.error( "Error in getting ONOS hosts from ONOS" + 1546 controllerStr ) 1547 consistentHostsResult = main.FALSE 1548 main.log.warn( "ONOS" + controllerStr + 1549 " hosts response: " + 1550 repr( hosts[ controller ] ) ) 1551 utilities.assert_equals( 1552 expect=main.TRUE, 1553 actual=consistentHostsResult, 1554 onpass="Hosts view is consistent across all ONOS nodes", 1555 onfail="ONOS nodes have different views of hosts" ) 1556 1557 main.step( "Each host has an IP address" ) 1558 ipResult = main.TRUE 1559 for controller in range( 0, len( hosts ) ): 1560 controllerStr = str( controller + 1 ) 1561 for host in hosts[ controller ]: 1562 if not host.get( 'ipAddresses', [ ] ): 1563 main.log.error( "DEBUG:Error with host ips on controller" + 1564 controllerStr + ": " + str( host ) ) 1565 ipResult = main.FALSE 1566 utilities.assert_equals( 1567 expect=main.TRUE, 1568 actual=ipResult, 1569 onpass="The ips of the hosts aren't empty", 1570 onfail="The ip of at least one host is missing" ) 1571 1572 # Strongly connected clusters of devices 1573 main.step( "Cluster view is consistent across ONOS nodes" ) 1574 consistentClustersResult = main.TRUE 1575 for controller in range( len( clusters ) ): 1576 controllerStr = str( controller + 1 ) 1577 if "Error" not in clusters[ controller ]: 1578 if clusters[ controller ] == clusters[ 0 ]: 1579 continue 1580 else: # clusters not consistent 1581 main.log.error( "clusters from ONOS" + controllerStr + 1582 " is inconsistent with ONOS1" ) 1583 consistentClustersResult = main.FALSE 1584 1585 else: 1586 main.log.error( "Error in getting dataplane clusters " + 1587 "from ONOS" + controllerStr ) 1588 consistentClustersResult = main.FALSE 1589 main.log.warn( "ONOS" + controllerStr + 1590 " clusters response: " + 1591 repr( clusters[ controller ] ) ) 1592 utilities.assert_equals( 1593 expect=main.TRUE, 1594 actual=consistentClustersResult, 1595 onpass="Clusters view is consistent across all ONOS nodes", 1596 onfail="ONOS nodes have different views of clusters" ) 1597 # there should always only be one cluster 1598 main.step( "Cluster view correct across ONOS nodes" ) 1599 try: 1600 numClusters = len( json.loads( clusters[ 0 ] ) ) 1601 except ( ValueError, TypeError ): 1602 main.log.exception( "Error parsing clusters[0]: " + 1603 repr( clusters[ 0 ] ) ) 1604 clusterResults = main.FALSE 1605 if numClusters == 1: 1606 clusterResults = main.TRUE 1607 utilities.assert_equals( 1608 expect=1, 1609 actual=numClusters, 1610 onpass="ONOS shows 1 SCC", 1611 onfail="ONOS shows " + str( numClusters ) + " SCCs" ) 1612 1613 main.step( "Comparing ONOS topology to MN" ) 1614 devicesResults = main.TRUE 1615 linksResults = main.TRUE 1616 hostsResults = main.TRUE 1617 mnSwitches = main.Mininet1.getSwitches() 1618 mnLinks = main.Mininet1.getLinks() 1619 mnHosts = main.Mininet1.getHosts() 1620 for controller in range( main.numCtrls ): 1621 controllerStr = str( controller + 1 ) 1622 if devices[ controller ] and ports[ controller ] and\ 1623 "Error" not in devices[ controller ] and\ 1624 "Error" not in ports[ controller ]: 1625 currentDevicesResult = main.Mininet1.compareSwitches( 1626 mnSwitches, 1627 json.loads( devices[ controller ] ), 1628 json.loads( ports[ controller ] ) ) 1629 else: 1630 currentDevicesResult = main.FALSE 1631 utilities.assert_equals( expect=main.TRUE, 1632 actual=currentDevicesResult, 1633 onpass="ONOS" + controllerStr + 1634 " Switches view is correct", 1635 onfail="ONOS" + controllerStr + 1636 " Switches view is incorrect" ) 1637 if links[ controller ] and "Error" not in links[ controller ]: 1638 currentLinksResult = main.Mininet1.compareLinks( 1639 mnSwitches, mnLinks, 1640 json.loads( links[ controller ] ) ) 1641 else: 1642 currentLinksResult = main.FALSE 1643 utilities.assert_equals( expect=main.TRUE, 1644 actual=currentLinksResult, 1645 onpass="ONOS" + controllerStr + 1646 " links view is correct", 1647 onfail="ONOS" + controllerStr + 1648 " links view is incorrect" ) 1649 1650 if hosts[ controller ] or "Error" not in hosts[ controller ]: 1651 currentHostsResult = main.Mininet1.compareHosts( 1652 mnHosts, 1653 hosts[ controller ] ) 1654 else: 1655 currentHostsResult = main.FALSE 1656 utilities.assert_equals( expect=main.TRUE, 1657 actual=currentHostsResult, 1658 onpass="ONOS" + controllerStr + 1659 " hosts exist in Mininet", 1660 onfail="ONOS" + controllerStr + 1661 " hosts don't match Mininet" ) 1662 1663 devicesResults = devicesResults and currentDevicesResult 1664 linksResults = linksResults and currentLinksResult 1665 hostsResults = hostsResults and currentHostsResult 1666 1667 main.step( "Device information is correct" ) 1668 utilities.assert_equals( 1669 expect=main.TRUE, 1670 actual=devicesResults, 1671 onpass="Device information is correct", 1672 onfail="Device information is incorrect" ) 1673 1674 main.step( "Links are correct" ) 1675 utilities.assert_equals( 1676 expect=main.TRUE, 1677 actual=linksResults, 1678 onpass="Link are correct", 1679 onfail="Links are incorrect" ) 1680 1681 main.step( "Hosts are correct" ) 1682 utilities.assert_equals( 1683 expect=main.TRUE, 1684 actual=hostsResults, 1685 onpass="Hosts are correct", 1686 onfail="Hosts are incorrect" )
1687
1688 - def CASE6( self, main ):
1689 """ 1690 The Failure case. 1691 """ 1692 import time 1693 assert main.numCtrls, "main.numCtrls not defined" 1694 assert main, "main not defined" 1695 assert utilities.assert_equals, "utilities.assert_equals not defined" 1696 assert main.CLIs, "main.CLIs not defined" 1697 assert main.nodes, "main.nodes not defined" 1698 try: 1699 labels 1700 except NameError: 1701 main.log.error( "labels not defined, setting to []" ) 1702 global labels 1703 labels = [] 1704 try: 1705 data 1706 except NameError: 1707 main.log.error( "data not defined, setting to []" ) 1708 global data 1709 data = [] 1710 # Reset non-persistent variables 1711 try: 1712 iCounterValue = 0 1713 except NameError: 1714 main.log.error( "iCounterValue not defined, setting to 0" ) 1715 iCounterValue = 0 1716 1717 main.case( "Restart entire ONOS cluster" ) 1718 1719 main.step( "Killing ONOS nodes" ) 1720 killResults = main.TRUE 1721 killTime = time.time() 1722 for node in main.nodes: 1723 killed = main.ONOSbench.onosKill( node.ip_address ) 1724 killResults = killResults and killed 1725 utilities.assert_equals( expect=main.TRUE, actual=killResults, 1726 onpass="ONOS nodes killed", 1727 onfail="ONOS kill unsuccessful" ) 1728 1729 main.step( "Checking if ONOS is up yet" ) 1730 for i in range( 2 ): 1731 onosIsupResult = main.TRUE 1732 for node in main.nodes: 1733 started = main.ONOSbench.isup( node.ip_address ) 1734 if not started: 1735 main.log.error( node.name + " didn't start!" ) 1736 onosIsupResult = onosIsupResult and started 1737 if onosIsupResult == main.TRUE: 1738 break 1739 utilities.assert_equals( expect=main.TRUE, actual=onosIsupResult, 1740 onpass="ONOS restarted", 1741 onfail="ONOS restart NOT successful" ) 1742 1743 main.log.step( "Starting ONOS CLI sessions" ) 1744 cliResults = main.TRUE 1745 threads = [] 1746 for i in range( main.numCtrls ): 1747 t = main.Thread( target=main.CLIs[i].startOnosCli, 1748 name="startOnosCli-" + str( i ), 1749 args=[main.nodes[i].ip_address] ) 1750 threads.append( t ) 1751 t.start() 1752 1753 for t in threads: 1754 t.join() 1755 cliResults = cliResults and t.result 1756 utilities.assert_equals( expect=main.TRUE, actual=cliResults, 1757 onpass="ONOS cli started", 1758 onfail="ONOS clis did not restart" ) 1759 1760 # Grab the time of restart so we chan check how long the gossip 1761 # protocol has had time to work 1762 main.restartTime = time.time() - killTime 1763 main.log.debug( "Restart time: " + str( main.restartTime ) ) 1764 labels.append( "Restart" ) 1765 data.append( str( main.restartTime ) ) 1766 1767 # FIXME: revisit test plan for election with madan 1768 # Rerun for election on restarted nodes 1769 runResults = main.TRUE 1770 for cli in main.CLIs: 1771 run = main.CLIs[0].electionTestRun() 1772 if run != main.TRUE: 1773 main.log.error( "Error running for election on " + cli.name ) 1774 runResults = runResults and run 1775 utilities.assert_equals( expect=main.TRUE, actual=runResults, 1776 onpass="Reran for election", 1777 onfail="Failed to rerun for election" ) 1778 1779 # TODO: Make this configurable 1780 time.sleep( 60 ) 1781 main.log.debug( main.CLIs[0].nodes( jsonFormat=False ) ) 1782 main.log.debug( main.CLIs[0].leaders( jsonFormat=False ) ) 1783 main.log.debug( main.CLIs[0].partitions( jsonFormat=False ) )
1784
1785 - def CASE7( self, main ):
1786 """ 1787 Check state after ONOS failure 1788 """ 1789 import json 1790 assert main.numCtrls, "main.numCtrls not defined" 1791 assert main, "main not defined" 1792 assert utilities.assert_equals, "utilities.assert_equals not defined" 1793 assert main.CLIs, "main.CLIs not defined" 1794 assert main.nodes, "main.nodes not defined" 1795 main.case( "Running ONOS Constant State Tests" ) 1796 1797 main.step( "Check that each switch has a master" ) 1798 # Assert that each device has a master 1799 rolesNotNull = main.TRUE 1800 threads = [] 1801 for i in range( main.numCtrls ): 1802 t = main.Thread( target=main.CLIs[i].rolesNotNull, 1803 name="rolesNotNull-" + str( i ), 1804 args=[ ] ) 1805 threads.append( t ) 1806 t.start() 1807 1808 for t in threads: 1809 t.join() 1810 rolesNotNull = rolesNotNull and t.result 1811 utilities.assert_equals( 1812 expect=main.TRUE, 1813 actual=rolesNotNull, 1814 onpass="Each device has a master", 1815 onfail="Some devices don't have a master assigned" ) 1816 1817 main.step( "Read device roles from ONOS" ) 1818 ONOSMastership = [] 1819 mastershipCheck = main.FALSE 1820 consistentMastership = True 1821 rolesResults = True 1822 threads = [] 1823 for i in range( main.numCtrls ): 1824 t = main.Thread( target=main.CLIs[i].roles, 1825 name="roles-" + str( i ), 1826 args=[] ) 1827 threads.append( t ) 1828 t.start() 1829 1830 for t in threads: 1831 t.join() 1832 ONOSMastership.append( t.result ) 1833 1834 for i in range( main.numCtrls ): 1835 if not ONOSMastership[i] or "Error" in ONOSMastership[i]: 1836 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1837 " roles" ) 1838 main.log.warn( 1839 "ONOS" + str( i + 1 ) + " mastership response: " + 1840 repr( ONOSMastership[i] ) ) 1841 rolesResults = False 1842 utilities.assert_equals( 1843 expect=True, 1844 actual=rolesResults, 1845 onpass="No error in reading roles output", 1846 onfail="Error in reading roles from ONOS" ) 1847 1848 main.step( "Check for consistency in roles from each controller" ) 1849 if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ): 1850 main.log.info( 1851 "Switch roles are consistent across all ONOS nodes" ) 1852 else: 1853 consistentMastership = False 1854 utilities.assert_equals( 1855 expect=True, 1856 actual=consistentMastership, 1857 onpass="Switch roles are consistent across all ONOS nodes", 1858 onfail="ONOS nodes have different views of switch roles" ) 1859 1860 if rolesResults and not consistentMastership: 1861 for i in range( main.numCtrls ): 1862 main.log.warn( 1863 "ONOS" + str( i + 1 ) + " roles: ", 1864 json.dumps( 1865 json.loads( ONOSMastership[ i ] ), 1866 sort_keys=True, 1867 indent=4, 1868 separators=( ',', ': ' ) ) ) 1869 elif rolesResults and not consistentMastership: 1870 mastershipCheck = main.TRUE 1871 1872 ''' 1873 description2 = "Compare switch roles from before failure" 1874 main.step( description2 ) 1875 try: 1876 currentJson = json.loads( ONOSMastership[0] ) 1877 oldJson = json.loads( mastershipState ) 1878 except ( ValueError, TypeError ): 1879 main.log.exception( "Something is wrong with parsing " + 1880 "ONOSMastership[0] or mastershipState" ) 1881 main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) ) 1882 main.log.error( "mastershipState" + repr( mastershipState ) ) 1883 main.cleanup() 1884 main.exit() 1885 mastershipCheck = main.TRUE 1886 for i in range( 1, 29 ): 1887 switchDPID = str( 1888 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) ) 1889 current = [ switch[ 'master' ] for switch in currentJson 1890 if switchDPID in switch[ 'id' ] ] 1891 old = [ switch[ 'master' ] for switch in oldJson 1892 if switchDPID in switch[ 'id' ] ] 1893 if current == old: 1894 mastershipCheck = mastershipCheck and main.TRUE 1895 else: 1896 main.log.warn( "Mastership of switch %s changed" % switchDPID ) 1897 mastershipCheck = main.FALSE 1898 utilities.assert_equals( 1899 expect=main.TRUE, 1900 actual=mastershipCheck, 1901 onpass="Mastership of Switches was not changed", 1902 onfail="Mastership of some switches changed" ) 1903 ''' 1904 # NOTE: we expect mastership to change on controller failure 1905 1906 main.step( "Get the intents and compare across all nodes" ) 1907 ONOSIntents = [] 1908 intentCheck = main.FALSE 1909 consistentIntents = True 1910 intentsResults = True 1911 threads = [] 1912 for i in range( main.numCtrls ): 1913 t = main.Thread( target=main.CLIs[i].intents, 1914 name="intents-" + str( i ), 1915 args=[], 1916 kwargs={ 'jsonFormat': True } ) 1917 threads.append( t ) 1918 t.start() 1919 1920 for t in threads: 1921 t.join() 1922 ONOSIntents.append( t.result ) 1923 1924 for i in range( main.numCtrls ): 1925 if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]: 1926 main.log.error( "Error in getting ONOS" + str( i + 1 ) + 1927 " intents" ) 1928 main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " + 1929 repr( ONOSIntents[ i ] ) ) 1930 intentsResults = False 1931 utilities.assert_equals( 1932 expect=True, 1933 actual=intentsResults, 1934 onpass="No error in reading intents output", 1935 onfail="Error in reading intents from ONOS" ) 1936 1937 main.step( "Check for consistency in Intents from each controller" ) 1938 if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ): 1939 main.log.info( "Intents are consistent across all ONOS " + 1940 "nodes" ) 1941 else: 1942 consistentIntents = False 1943 1944 # Try to make it easy to figure out what is happening 1945 # 1946 # Intent ONOS1 ONOS2 ... 1947 # 0x01 INSTALLED INSTALLING 1948 # ... ... ... 1949 # ... ... ... 1950 title = " ID" 1951 for n in range( main.numCtrls ): 1952 title += " " * 10 + "ONOS" + str( n + 1 ) 1953 main.log.warn( title ) 1954 # get all intent keys in the cluster 1955 keys = [] 1956 for nodeStr in ONOSIntents: 1957 node = json.loads( nodeStr ) 1958 for intent in node: 1959 keys.append( intent.get( 'id' ) ) 1960 keys = set( keys ) 1961 for key in keys: 1962 row = "%-13s" % key 1963 for nodeStr in ONOSIntents: 1964 node = json.loads( nodeStr ) 1965 for intent in node: 1966 if intent.get( 'id' ) == key: 1967 row += "%-15s" % intent.get( 'state' ) 1968 main.log.warn( row ) 1969 # End table view 1970 1971 utilities.assert_equals( 1972 expect=True, 1973 actual=consistentIntents, 1974 onpass="Intents are consistent across all ONOS nodes", 1975 onfail="ONOS nodes have different views of intents" ) 1976 intentStates = [] 1977 for node in ONOSIntents: # Iter through ONOS nodes 1978 nodeStates = [] 1979 # Iter through intents of a node 1980 try: 1981 for intent in json.loads( node ): 1982 nodeStates.append( intent[ 'state' ] ) 1983 except ( ValueError, TypeError ): 1984 main.log.exception( "Error in parsing intents" ) 1985 main.log.error( repr( node ) ) 1986 intentStates.append( nodeStates ) 1987 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ] 1988 main.log.info( dict( out ) ) 1989 1990 if intentsResults and not consistentIntents: 1991 for i in range( main.numCtrls ): 1992 main.log.warn( "ONOS" + str( i + 1 ) + " intents: " ) 1993 main.log.warn( json.dumps( 1994 json.loads( ONOSIntents[ i ] ), 1995 sort_keys=True, 1996 indent=4, 1997 separators=( ',', ': ' ) ) ) 1998 elif intentsResults and consistentIntents: 1999 intentCheck = main.TRUE 2000 2001 # NOTE: Store has no durability, so intents are lost across system 2002 # restarts 2003 """ 2004 main.step( "Compare current intents with intents before the failure" ) 2005 # NOTE: this requires case 5 to pass for intentState to be set. 2006 # maybe we should stop the test if that fails? 2007 sameIntents = main.FALSE 2008 if intentState and intentState == ONOSIntents[ 0 ]: 2009 sameIntents = main.TRUE 2010 main.log.info( "Intents are consistent with before failure" ) 2011 # TODO: possibly the states have changed? we may need to figure out 2012 # what the acceptable states are 2013 elif len( intentState ) == len( ONOSIntents[ 0 ] ): 2014 sameIntents = main.TRUE 2015 try: 2016 before = json.loads( intentState ) 2017 after = json.loads( ONOSIntents[ 0 ] ) 2018 for intent in before: 2019 if intent not in after: 2020 sameIntents = main.FALSE 2021 main.log.debug( "Intent is not currently in ONOS " + 2022 "(at least in the same form):" ) 2023 main.log.debug( json.dumps( intent ) ) 2024 except ( ValueError, TypeError ): 2025 main.log.exception( "Exception printing intents" ) 2026 main.log.debug( repr( ONOSIntents[0] ) ) 2027 main.log.debug( repr( intentState ) ) 2028 if sameIntents == main.FALSE: 2029 try: 2030 main.log.debug( "ONOS intents before: " ) 2031 main.log.debug( json.dumps( json.loads( intentState ), 2032 sort_keys=True, indent=4, 2033 separators=( ',', ': ' ) ) ) 2034 main.log.debug( "Current ONOS intents: " ) 2035 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ), 2036 sort_keys=True, indent=4, 2037 separators=( ',', ': ' ) ) ) 2038 except ( ValueError, TypeError ): 2039 main.log.exception( "Exception printing intents" ) 2040 main.log.debug( repr( ONOSIntents[0] ) ) 2041 main.log.debug( repr( intentState ) ) 2042 utilities.assert_equals( 2043 expect=main.TRUE, 2044 actual=sameIntents, 2045 onpass="Intents are consistent with before failure", 2046 onfail="The Intents changed during failure" ) 2047 intentCheck = intentCheck and sameIntents 2048 """ 2049 main.step( "Get the OF Table entries and compare to before " + 2050 "component failure" ) 2051 FlowTables = main.TRUE 2052 flows2 = [] 2053 for i in range( 28 ): 2054 main.log.info( "Checking flow table on s" + str( i + 1 ) ) 2055 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) ) 2056 flows2.append( tmpFlows ) 2057 tempResult = main.Mininet1.flowComp( 2058 flow1=flows[ i ], 2059 flow2=tmpFlows ) 2060 FlowTables = FlowTables and tempResult 2061 if FlowTables == main.FALSE: 2062 main.log.info( "Differences in flow table for switch: s" + 2063 str( i + 1 ) ) 2064 utilities.assert_equals( 2065 expect=main.TRUE, 2066 actual=FlowTables, 2067 onpass="No changes were found in the flow tables", 2068 onfail="Changes were found in the flow tables" ) 2069 2070 main.Mininet2.pingLongKill() 2071 ''' 2072 # main.step( "Check the continuous pings to ensure that no packets " + 2073 # "were dropped during component failure" ) 2074 main.Mininet2.pingKill( main.params[ 'TESTONUSER' ], 2075 main.params[ 'TESTONIP' ] ) 2076 LossInPings = main.FALSE 2077 # NOTE: checkForLoss returns main.FALSE with 0% packet loss 2078 for i in range( 8, 18 ): 2079 main.log.info( 2080 "Checking for a loss in pings along flow from s" + 2081 str( i ) ) 2082 LossInPings = main.Mininet2.checkForLoss( 2083 "/tmp/ping.h" + 2084 str( i ) ) or LossInPings 2085 if LossInPings == main.TRUE: 2086 main.log.info( "Loss in ping detected" ) 2087 elif LossInPings == main.ERROR: 2088 main.log.info( "There are multiple mininet process running" ) 2089 elif LossInPings == main.FALSE: 2090 main.log.info( "No Loss in the pings" ) 2091 main.log.info( "No loss of dataplane connectivity" ) 2092 # utilities.assert_equals( 2093 # expect=main.FALSE, 2094 # actual=LossInPings, 2095 # onpass="No Loss of connectivity", 2096 # onfail="Loss of dataplane connectivity detected" ) 2097 2098 # NOTE: Since intents are not persisted with IntnentStore, 2099 # we expect loss in dataplane connectivity 2100 LossInPings = main.FALSE 2101 ''' 2102 2103 main.step( "Leadership Election is still functional" ) 2104 # Test of LeadershipElection 2105 leaderList = [] 2106 leaderResult = main.TRUE 2107 for cli in main.CLIs: 2108 leaderN = cli.electionTestLeader() 2109 leaderList.append( leaderN ) 2110 if leaderN == main.FALSE: 2111 # error in response 2112 main.log.error( "Something is wrong with " + 2113 "electionTestLeader function, check the" + 2114 " error logs" ) 2115 leaderResult = main.FALSE 2116 elif leaderN is None: 2117 main.log.error( cli.name + 2118 " shows no leader for the election-app." ) 2119 leaderResult = main.FALSE 2120 if len( set( leaderList ) ) != 1: 2121 leaderResult = main.FALSE 2122 main.log.error( 2123 "Inconsistent view of leader for the election test app" ) 2124 # TODO: print the list 2125 utilities.assert_equals( 2126 expect=main.TRUE, 2127 actual=leaderResult, 2128 onpass="Leadership election passed", 2129 onfail="Something went wrong with Leadership election" )
2130
2131 - def CASE8( self, main ):
2132 """ 2133 Compare topo 2134 """ 2135 import json 2136 import time 2137 assert main.numCtrls, "main.numCtrls not defined" 2138 assert main, "main not defined" 2139 assert utilities.assert_equals, "utilities.assert_equals not defined" 2140 assert main.CLIs, "main.CLIs not defined" 2141 assert main.nodes, "main.nodes not defined" 2142 2143 main.case( "Compare ONOS Topology view to Mininet topology" ) 2144 main.caseExplanation = "Compare topology objects between Mininet" +\ 2145 " and ONOS" 2146 2147 main.step( "Comparing ONOS topology to MN" ) 2148 devicesResults = main.TRUE 2149 linksResults = main.TRUE 2150 hostsResults = main.TRUE 2151 hostAttachmentResults = True 2152 topoResult = main.FALSE 2153 elapsed = 0 2154 count = 0 2155 main.step( "Collecting topology information from ONOS" ) 2156 startTime = time.time() 2157 # Give time for Gossip to work 2158 while topoResult == main.FALSE and elapsed < 60: 2159 count += 1 2160 cliStart = time.time() 2161 devices = [] 2162 threads = [] 2163 for i in range( main.numCtrls ): 2164 t = main.Thread( target=main.CLIs[i].devices, 2165 name="devices-" + str( i ), 2166 args=[ ] ) 2167 threads.append( t ) 2168 t.start() 2169 2170 for t in threads: 2171 t.join() 2172 devices.append( t.result ) 2173 hosts = [] 2174 ipResult = main.TRUE 2175 threads = [] 2176 for i in range( main.numCtrls ): 2177 t = main.Thread( target=main.CLIs[i].hosts, 2178 name="hosts-" + str( i ), 2179 args=[ ] ) 2180 threads.append( t ) 2181 t.start() 2182 2183 for t in threads: 2184 t.join() 2185 try: 2186 hosts.append( json.loads( t.result ) ) 2187 except ( ValueError, TypeError ): 2188 main.log.exception( "Error parsing hosts results" ) 2189 main.log.error( repr( t.result ) ) 2190 for controller in range( 0, len( hosts ) ): 2191 controllerStr = str( controller + 1 ) 2192 for host in hosts[ controller ]: 2193 if host is None or host.get( 'ipAddresses', [] ) == []: 2194 main.log.error( 2195 "DEBUG:Error with host ipAddresses on controller" + 2196 controllerStr + ": " + str( host ) ) 2197 ipResult = main.FALSE 2198 ports = [] 2199 threads = [] 2200 for i in range( main.numCtrls ): 2201 t = main.Thread( target=main.CLIs[i].ports, 2202 name="ports-" + str( i ), 2203 args=[ ] ) 2204 threads.append( t ) 2205 t.start() 2206 2207 for t in threads: 2208 t.join() 2209 ports.append( t.result ) 2210 links = [] 2211 threads = [] 2212 for i in range( main.numCtrls ): 2213 t = main.Thread( target=main.CLIs[i].links, 2214 name="links-" + str( i ), 2215 args=[ ] ) 2216 threads.append( t ) 2217 t.start() 2218 2219 for t in threads: 2220 t.join() 2221 links.append( t.result ) 2222 clusters = [] 2223 threads = [] 2224 for i in range( main.numCtrls ): 2225 t = main.Thread( target=main.CLIs[i].clusters, 2226 name="clusters-" + str( i ), 2227 args=[ ] ) 2228 threads.append( t ) 2229 t.start() 2230 2231 for t in threads: 2232 t.join() 2233 clusters.append( t.result ) 2234 2235 elapsed = time.time() - startTime 2236 cliTime = time.time() - cliStart 2237 print "Elapsed time: " + str( elapsed ) 2238 print "CLI time: " + str( cliTime ) 2239 2240 mnSwitches = main.Mininet1.getSwitches() 2241 mnLinks = main.Mininet1.getLinks() 2242 mnHosts = main.Mininet1.getHosts() 2243 for controller in range( main.numCtrls ): 2244 controllerStr = str( controller + 1 ) 2245 if devices[ controller ] and ports[ controller ] and\ 2246 "Error" not in devices[ controller ] and\ 2247 "Error" not in ports[ controller ]: 2248 2249 currentDevicesResult = main.Mininet1.compareSwitches( 2250 mnSwitches, 2251 json.loads( devices[ controller ] ), 2252 json.loads( ports[ controller ] ) ) 2253 else: 2254 currentDevicesResult = main.FALSE 2255 utilities.assert_equals( expect=main.TRUE, 2256 actual=currentDevicesResult, 2257 onpass="ONOS" + controllerStr + 2258 " Switches view is correct", 2259 onfail="ONOS" + controllerStr + 2260 " Switches view is incorrect" ) 2261 2262 if links[ controller ] and "Error" not in links[ controller ]: 2263 currentLinksResult = main.Mininet1.compareLinks( 2264 mnSwitches, mnLinks, 2265 json.loads( links[ controller ] ) ) 2266 else: 2267 currentLinksResult = main.FALSE 2268 utilities.assert_equals( expect=main.TRUE, 2269 actual=currentLinksResult, 2270 onpass="ONOS" + controllerStr + 2271 " links view is correct", 2272 onfail="ONOS" + controllerStr + 2273 " links view is incorrect" ) 2274 2275 if hosts[ controller ] or "Error" not in hosts[ controller ]: 2276 currentHostsResult = main.Mininet1.compareHosts( 2277 mnHosts, 2278 hosts[ controller ] ) 2279 else: 2280 currentHostsResult = main.FALSE 2281 utilities.assert_equals( expect=main.TRUE, 2282 actual=currentHostsResult, 2283 onpass="ONOS" + controllerStr + 2284 " hosts exist in Mininet", 2285 onfail="ONOS" + controllerStr + 2286 " hosts don't match Mininet" ) 2287 # CHECKING HOST ATTACHMENT POINTS 2288 hostAttachment = True 2289 noHosts = False 2290 # FIXME: topo-HA/obelisk specific mappings: 2291 # key is mac and value is dpid 2292 mappings = {} 2293 for i in range( 1, 29 ): # hosts 1 through 28 2294 # set up correct variables: 2295 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2) 2296 if i == 1: 2297 deviceId = "1000".zfill(16) 2298 elif i == 2: 2299 deviceId = "2000".zfill(16) 2300 elif i == 3: 2301 deviceId = "3000".zfill(16) 2302 elif i == 4: 2303 deviceId = "3004".zfill(16) 2304 elif i == 5: 2305 deviceId = "5000".zfill(16) 2306 elif i == 6: 2307 deviceId = "6000".zfill(16) 2308 elif i == 7: 2309 deviceId = "6007".zfill(16) 2310 elif i >= 8 and i <= 17: 2311 dpid = '3' + str( i ).zfill( 3 ) 2312 deviceId = dpid.zfill(16) 2313 elif i >= 18 and i <= 27: 2314 dpid = '6' + str( i ).zfill( 3 ) 2315 deviceId = dpid.zfill(16) 2316 elif i == 28: 2317 deviceId = "2800".zfill(16) 2318 mappings[ macId ] = deviceId 2319 if hosts[ controller ] or "Error" not in hosts[ controller ]: 2320 if hosts[ controller ] == []: 2321 main.log.warn( "There are no hosts discovered" ) 2322 noHosts = True 2323 else: 2324 for host in hosts[ controller ]: 2325 mac = None 2326 location = None 2327 device = None 2328 port = None 2329 try: 2330 mac = host.get( 'mac' ) 2331 assert mac, "mac field could not be found for this host object" 2332 2333 location = host.get( 'location' ) 2334 assert location, "location field could not be found for this host object" 2335 2336 # Trim the protocol identifier off deviceId 2337 device = str( location.get( 'elementId' ) ).split(':')[1] 2338 assert device, "elementId field could not be found for this host location object" 2339 2340 port = location.get( 'port' ) 2341 assert port, "port field could not be found for this host location object" 2342 2343 # Now check if this matches where they should be 2344 if mac and device and port: 2345 if str( port ) != "1": 2346 main.log.error( "The attachment port is incorrect for " + 2347 "host " + str( mac ) + 2348 ". Expected: 1 Actual: " + str( port) ) 2349 hostAttachment = False 2350 if device != mappings[ str( mac ) ]: 2351 main.log.error( "The attachment device is incorrect for " + 2352 "host " + str( mac ) + 2353 ". Expected: " + mappings[ str( mac ) ] + 2354 " Actual: " + device ) 2355 hostAttachment = False 2356 else: 2357 hostAttachment = False 2358 except AssertionError: 2359 main.log.exception( "Json object not as expected" ) 2360 main.log.error( repr( host ) ) 2361 hostAttachment = False 2362 else: 2363 main.log.error( "No hosts json output or \"Error\"" + 2364 " in output. hosts = " + 2365 repr( hosts[ controller ] ) ) 2366 if noHosts is False: 2367 # TODO: Find a way to know if there should be hosts in a 2368 # given point of the test 2369 hostAttachment = True 2370 2371 # END CHECKING HOST ATTACHMENT POINTS 2372 devicesResults = devicesResults and currentDevicesResult 2373 linksResults = linksResults and currentLinksResult 2374 hostsResults = hostsResults and currentHostsResult 2375 hostAttachmentResults = hostAttachmentResults and\ 2376 hostAttachment 2377 topoResult = ( devicesResults and linksResults 2378 and hostsResults and ipResult and 2379 hostAttachmentResults ) 2380 2381 # Compare json objects for hosts and dataplane clusters 2382 2383 # hosts 2384 main.step( "Hosts view is consistent across all ONOS nodes" ) 2385 consistentHostsResult = main.TRUE 2386 for controller in range( len( hosts ) ): 2387 controllerStr = str( controller + 1 ) 2388 if "Error" not in hosts[ controller ]: 2389 if hosts[ controller ] == hosts[ 0 ]: 2390 continue 2391 else: # hosts not consistent 2392 main.log.error( "hosts from ONOS" + controllerStr + 2393 " is inconsistent with ONOS1" ) 2394 main.log.warn( repr( hosts[ controller ] ) ) 2395 consistentHostsResult = main.FALSE 2396 2397 else: 2398 main.log.error( "Error in getting ONOS hosts from ONOS" + 2399 controllerStr ) 2400 consistentHostsResult = main.FALSE 2401 main.log.warn( "ONOS" + controllerStr + 2402 " hosts response: " + 2403 repr( hosts[ controller ] ) ) 2404 utilities.assert_equals( 2405 expect=main.TRUE, 2406 actual=consistentHostsResult, 2407 onpass="Hosts view is consistent across all ONOS nodes", 2408 onfail="ONOS nodes have different views of hosts" ) 2409 2410 main.step( "Hosts information is correct" ) 2411 hostsResults = hostsResults and ipResult 2412 utilities.assert_equals( 2413 expect=main.TRUE, 2414 actual=hostsResults, 2415 onpass="Host information is correct", 2416 onfail="Host information is incorrect" ) 2417 2418 main.step( "Host attachment points to the network" ) 2419 utilities.assert_equals( 2420 expect=True, 2421 actual=hostAttachmentResults, 2422 onpass="Hosts are correctly attached to the network", 2423 onfail="ONOS did not correctly attach hosts to the network" ) 2424 2425 # Strongly connected clusters of devices 2426 main.step( "Clusters view is consistent across all ONOS nodes" ) 2427 consistentClustersResult = main.TRUE 2428 for controller in range( len( clusters ) ): 2429 controllerStr = str( controller + 1 ) 2430 if "Error" not in clusters[ controller ]: 2431 if clusters[ controller ] == clusters[ 0 ]: 2432 continue 2433 else: # clusters not consistent 2434 main.log.error( "clusters from ONOS" + 2435 controllerStr + 2436 " is inconsistent with ONOS1" ) 2437 consistentClustersResult = main.FALSE 2438 2439 else: 2440 main.log.error( "Error in getting dataplane clusters " + 2441 "from ONOS" + controllerStr ) 2442 consistentClustersResult = main.FALSE 2443 main.log.warn( "ONOS" + controllerStr + 2444 " clusters response: " + 2445 repr( clusters[ controller ] ) ) 2446 utilities.assert_equals( 2447 expect=main.TRUE, 2448 actual=consistentClustersResult, 2449 onpass="Clusters view is consistent across all ONOS nodes", 2450 onfail="ONOS nodes have different views of clusters" ) 2451 2452 main.step( "There is only one SCC" ) 2453 # there should always only be one cluster 2454 try: 2455 numClusters = len( json.loads( clusters[ 0 ] ) ) 2456 except ( ValueError, TypeError ): 2457 main.log.exception( "Error parsing clusters[0]: " + 2458 repr( clusters[0] ) ) 2459 clusterResults = main.FALSE 2460 if numClusters == 1: 2461 clusterResults = main.TRUE 2462 utilities.assert_equals( 2463 expect=1, 2464 actual=numClusters, 2465 onpass="ONOS shows 1 SCC", 2466 onfail="ONOS shows " + str( numClusters ) + " SCCs" ) 2467 2468 topoResult = ( devicesResults and linksResults 2469 and hostsResults and consistentHostsResult 2470 and consistentClustersResult and clusterResults 2471 and ipResult and hostAttachmentResults ) 2472 2473 topoResult = topoResult and int( count <= 2 ) 2474 note = "note it takes about " + str( int( cliTime ) ) + \ 2475 " seconds for the test to make all the cli calls to fetch " +\ 2476 "the topology from each ONOS instance" 2477 main.log.info( 2478 "Very crass estimate for topology discovery/convergence( " + 2479 str( note ) + " ): " + str( elapsed ) + " seconds, " + 2480 str( count ) + " tries" ) 2481 2482 main.step( "Device information is correct" ) 2483 utilities.assert_equals( 2484 expect=main.TRUE, 2485 actual=devicesResults, 2486 onpass="Device information is correct", 2487 onfail="Device information is incorrect" ) 2488 2489 main.step( "Links are correct" ) 2490 utilities.assert_equals( 2491 expect=main.TRUE, 2492 actual=linksResults, 2493 onpass="Link are correct", 2494 onfail="Links are incorrect" ) 2495 2496 # FIXME: move this to an ONOS state case 2497 main.step( "Checking ONOS nodes" ) 2498 nodesOutput = [] 2499 nodeResults = main.TRUE 2500 threads = [] 2501 for i in range( main.numCtrls ): 2502 t = main.Thread( target=main.CLIs[i].nodes, 2503 name="nodes-" + str( i ), 2504 args=[ ] ) 2505 threads.append( t ) 2506 t.start() 2507 2508 for t in threads: 2509 t.join() 2510 nodesOutput.append( t.result ) 2511 ips = [ node.ip_address for node in main.nodes ] 2512 for i in nodesOutput: 2513 try: 2514 current = json.loads( i ) 2515 for node in current: 2516 currentResult = main.FALSE 2517 if node['ip'] in ips: # node in nodes() output is in cell 2518 if node['state'] == 'ACTIVE': 2519 currentResult = main.TRUE 2520 else: 2521 main.log.error( "Error in ONOS node availability" ) 2522 main.log.error( 2523 json.dumps( current, 2524 sort_keys=True, 2525 indent=4, 2526 separators=( ',', ': ' ) ) ) 2527 break 2528 nodeResults = nodeResults and currentResult 2529 except ( ValueError, TypeError ): 2530 main.log.error( "Error parsing nodes output" ) 2531 main.log.warn( repr( i ) ) 2532 utilities.assert_equals( expect=main.TRUE, actual=nodeResults, 2533 onpass="Nodes check successful", 2534 onfail="Nodes check NOT successful" )
2535
2536 - def CASE9( self, main ):
2537 """ 2538 Link s3-s28 down 2539 """ 2540 import time 2541 assert main.numCtrls, "main.numCtrls not defined" 2542 assert main, "main not defined" 2543 assert utilities.assert_equals, "utilities.assert_equals not defined" 2544 assert main.CLIs, "main.CLIs not defined" 2545 assert main.nodes, "main.nodes not defined" 2546 # NOTE: You should probably run a topology check after this 2547 2548 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] ) 2549 2550 description = "Turn off a link to ensure that Link Discovery " +\ 2551 "is working properly" 2552 main.case( description ) 2553 2554 main.step( "Kill Link between s3 and s28" ) 2555 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" ) 2556 main.log.info( "Waiting " + str( linkSleep ) + 2557 " seconds for link down to be discovered" ) 2558 time.sleep( linkSleep ) 2559 utilities.assert_equals( expect=main.TRUE, actual=LinkDown, 2560 onpass="Link down successful", 2561 onfail="Failed to bring link down" )
2562 # TODO do some sort of check here 2563
2564 - def CASE10( self, main ):
2565 """ 2566 Link s3-s28 up 2567 """ 2568 import time 2569 assert main.numCtrls, "main.numCtrls not defined" 2570 assert main, "main not defined" 2571 assert utilities.assert_equals, "utilities.assert_equals not defined" 2572 assert main.CLIs, "main.CLIs not defined" 2573 assert main.nodes, "main.nodes not defined" 2574 # NOTE: You should probably run a topology check after this 2575 2576 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] ) 2577 2578 description = "Restore a link to ensure that Link Discovery is " + \ 2579 "working properly" 2580 main.case( description ) 2581 2582 main.step( "Bring link between s3 and s28 back up" ) 2583 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" ) 2584 main.log.info( "Waiting " + str( linkSleep ) + 2585 " seconds for link up to be discovered" ) 2586 time.sleep( linkSleep ) 2587 utilities.assert_equals( expect=main.TRUE, actual=LinkUp, 2588 onpass="Link up successful", 2589 onfail="Failed to bring link up" )
2590 # TODO do some sort of check here 2591
2592 - def CASE11( self, main ):
2593 """ 2594 Switch Down 2595 """ 2596 # NOTE: You should probably run a topology check after this 2597 import time 2598 assert main.numCtrls, "main.numCtrls not defined" 2599 assert main, "main not defined" 2600 assert utilities.assert_equals, "utilities.assert_equals not defined" 2601 assert main.CLIs, "main.CLIs not defined" 2602 assert main.nodes, "main.nodes not defined" 2603 2604 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) 2605 2606 description = "Killing a switch to ensure it is discovered correctly" 2607 main.case( description ) 2608 switch = main.params[ 'kill' ][ 'switch' ] 2609 switchDPID = main.params[ 'kill' ][ 'dpid' ] 2610 2611 # TODO: Make this switch parameterizable 2612 main.step( "Kill " + switch ) 2613 main.log.info( "Deleting " + switch ) 2614 main.Mininet1.delSwitch( switch ) 2615 main.log.info( "Waiting " + str( switchSleep ) + 2616 " seconds for switch down to be discovered" ) 2617 time.sleep( switchSleep ) 2618 device = main.ONOScli1.getDevice( dpid=switchDPID ) 2619 # Peek at the deleted switch 2620 main.log.warn( str( device ) ) 2621 result = main.FALSE 2622 if device and device[ 'available' ] is False: 2623 result = main.TRUE 2624 utilities.assert_equals( expect=main.TRUE, actual=result, 2625 onpass="Kill switch successful", 2626 onfail="Failed to kill switch?" )
2627
2628 - def CASE12( self, main ):
2629 """ 2630 Switch Up 2631 """ 2632 # NOTE: You should probably run a topology check after this 2633 import time 2634 assert main.numCtrls, "main.numCtrls not defined" 2635 assert main, "main not defined" 2636 assert utilities.assert_equals, "utilities.assert_equals not defined" 2637 assert main.CLIs, "main.CLIs not defined" 2638 assert main.nodes, "main.nodes not defined" 2639 assert ONOS1Port, "ONOS1Port not defined" 2640 assert ONOS2Port, "ONOS2Port not defined" 2641 assert ONOS3Port, "ONOS3Port not defined" 2642 assert ONOS4Port, "ONOS4Port not defined" 2643 assert ONOS5Port, "ONOS5Port not defined" 2644 assert ONOS6Port, "ONOS6Port not defined" 2645 assert ONOS7Port, "ONOS7Port not defined" 2646 2647 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) 2648 switch = main.params[ 'kill' ][ 'switch' ] 2649 switchDPID = main.params[ 'kill' ][ 'dpid' ] 2650 links = main.params[ 'kill' ][ 'links' ].split() 2651 description = "Adding a switch to ensure it is discovered correctly" 2652 main.case( description ) 2653 2654 main.step( "Add back " + switch ) 2655 main.Mininet1.addSwitch( switch, dpid=switchDPID ) 2656 for peer in links: 2657 main.Mininet1.addLink( switch, peer ) 2658 ipList = [] 2659 for i in range( main.numCtrls ): 2660 ipList.append( main.nodes[ i ].ip_address ) 2661 main.Mininet1.assignSwController( sw=switch, ip=ipList ) 2662 main.log.info( "Waiting " + str( switchSleep ) + 2663 " seconds for switch up to be discovered" ) 2664 time.sleep( switchSleep ) 2665 device = main.ONOScli1.getDevice( dpid=switchDPID ) 2666 # Peek at the deleted switch 2667 main.log.warn( str( device ) ) 2668 result = main.FALSE 2669 if device and device[ 'available' ]: 2670 result = main.TRUE 2671 utilities.assert_equals( expect=main.TRUE, actual=result, 2672 onpass="add switch successful", 2673 onfail="Failed to add switch?" )
2674
2675 - def CASE13( self, main ):
2676 """ 2677 Clean up 2678 """ 2679 import os 2680 import time 2681 assert main.numCtrls, "main.numCtrls not defined" 2682 assert main, "main not defined" 2683 assert utilities.assert_equals, "utilities.assert_equals not defined" 2684 assert main.CLIs, "main.CLIs not defined" 2685 assert main.nodes, "main.nodes not defined" 2686 2687 # printing colors to terminal 2688 colors = { 'cyan': '\033[96m', 'purple': '\033[95m', 2689 'blue': '\033[94m', 'green': '\033[92m', 2690 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' } 2691 main.case( "Test Cleanup" ) 2692 main.step( "Killing tcpdumps" ) 2693 main.Mininet2.stopTcpdump() 2694 2695 testname = main.TEST 2696 if main.params[ 'BACKUP' ] == "True": 2697 main.step( "Copying MN pcap and ONOS log files to test station" ) 2698 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ] 2699 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ] 2700 # NOTE: MN Pcap file is being saved to ~/packet_captures 2701 # scp this file as MN and TestON aren't necessarily the same vm 2702 # FIXME: scp 2703 # mn files 2704 # TODO: Load these from params 2705 # NOTE: must end in / 2706 logFolder = "/opt/onos/log/" 2707 logFiles = [ "karaf.log", "karaf.log.1" ] 2708 # NOTE: must end in / 2709 dstDir = "~/packet_captures/" 2710 for f in logFiles: 2711 for node in main.nodes: 2712 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address + 2713 ":" + logFolder + f + " " + 2714 teststationUser + "@" + 2715 teststationIP + ":" + 2716 dstDir + str( testname ) + 2717 "-" + node.name + "-" + f ) 2718 main.ONOSbench.handle.expect( "\$" ) 2719 2720 # std*.log's 2721 # NOTE: must end in / 2722 logFolder = "/opt/onos/var/" 2723 logFiles = [ "stderr.log", "stdout.log" ] 2724 # NOTE: must end in / 2725 dstDir = "~/packet_captures/" 2726 for f in logFiles: 2727 for node in main.nodes: 2728 main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address + 2729 ":" + logFolder + f + " " + 2730 teststationUser + "@" + 2731 teststationIP + ":" + 2732 dstDir + str( testname ) + 2733 "-" + node.name + "-" + f ) 2734 main.ONOSbench.handle.expect( "\$" ) 2735 # sleep so scp can finish 2736 time.sleep( 10 ) 2737 main.step( "Packing and rotating pcap archives" ) 2738 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) ) 2739 2740 main.step( "Stopping Mininet" ) 2741 mnResult = main.Mininet1.stopNet() 2742 utilities.assert_equals( expect=main.TRUE, actual=mnResult, 2743 onpass="Mininet stopped", 2744 onfail="MN cleanup NOT successful" ) 2745 2746 main.step( "Checking ONOS Logs for errors" ) 2747 for node in main.nodes: 2748 print colors[ 'purple' ] + "Checking logs for errors on " + \ 2749 node.name + ":" + colors[ 'end' ] 2750 print main.ONOSbench.checkLogs( node.ip_address, restart=True ) 2751 2752 try: 2753 timerLog = open( main.logdir + "/Timers.csv", 'w') 2754 main.log.error( ", ".join( labels ) + "\n" + ", ".join( data ) ) 2755 timerLog.write( ", ".join( labels ) + "\n" + ", ".join( data ) ) 2756 timerLog.close() 2757 except NameError, e: 2758 main.log.exception(e)
2759
2760 - def CASE14( self, main ):
2761 """ 2762 start election app on all onos nodes 2763 """ 2764 assert main.numCtrls, "main.numCtrls not defined" 2765 assert main, "main not defined" 2766 assert utilities.assert_equals, "utilities.assert_equals not defined" 2767 assert main.CLIs, "main.CLIs not defined" 2768 assert main.nodes, "main.nodes not defined" 2769 2770 main.case("Start Leadership Election app") 2771 main.step( "Install leadership election app" ) 2772 appResult = main.ONOScli1.activateApp( "org.onosproject.election" ) 2773 utilities.assert_equals( 2774 expect=main.TRUE, 2775 actual=appResult, 2776 onpass="Election app installed", 2777 onfail="Something went wrong with installing Leadership election" ) 2778 2779 main.step( "Run for election on each node" ) 2780 leaderResult = main.TRUE 2781 leaders = [] 2782 for cli in main.CLIs: 2783 cli.electionTestRun() 2784 for cli in main.CLIs: 2785 leader = cli.electionTestLeader() 2786 if leader is None or leader == main.FALSE: 2787 main.log.error( cli.name + ": Leader for the election app " + 2788 "should be an ONOS node, instead got '" + 2789 str( leader ) + "'" ) 2790 leaderResult = main.FALSE 2791 leaders.append( leader ) 2792 utilities.assert_equals( 2793 expect=main.TRUE, 2794 actual=leaderResult, 2795 onpass="Successfully ran for leadership", 2796 onfail="Failed to run for leadership" ) 2797 2798 main.step( "Check that each node shows the same leader" ) 2799 sameLeader = main.TRUE 2800 if len( set( leaders ) ) != 1: 2801 sameLeader = main.FALSE 2802 main.log.error( "Results of electionTestLeader is order of CLIs:" + 2803 str( leaders ) ) 2804 utilities.assert_equals( 2805 expect=main.TRUE, 2806 actual=sameLeader, 2807 onpass="Leadership is consistent for the election topic", 2808 onfail="Nodes have different leaders" )
2809
2810 - def CASE15( self, main ):
2811 """ 2812 Check that Leadership Election is still functional 2813 """ 2814 import time 2815 assert main.numCtrls, "main.numCtrls not defined" 2816 assert main, "main not defined" 2817 assert utilities.assert_equals, "utilities.assert_equals not defined" 2818 assert main.CLIs, "main.CLIs not defined" 2819 assert main.nodes, "main.nodes not defined" 2820 2821 leaderResult = main.TRUE 2822 description = "Check that Leadership Election is still functional" 2823 main.case( description ) 2824 # NOTE: Need to re-run since being a canidate is not persistant 2825 main.step( "Run for election on each node" ) 2826 leaderResult = main.TRUE 2827 leaders = [] 2828 for cli in main.CLIs: 2829 cli.electionTestRun() 2830 for cli in main.CLIs: 2831 leader = cli.electionTestLeader() 2832 if leader is None or leader == main.FALSE: 2833 main.log.error( cli.name + ": Leader for the election app " + 2834 "should be an ONOS node, instead got '" + 2835 str( leader ) + "'" ) 2836 leaderResult = main.FALSE 2837 leaders.append( leader ) 2838 utilities.assert_equals( 2839 expect=main.TRUE, 2840 actual=leaderResult, 2841 onpass="Successfully ran for leadership", 2842 onfail="Failed to run for leadership" ) 2843 2844 main.step( "Check that each node shows the same leader" ) 2845 sameLeader = main.TRUE 2846 if len( set( leaders ) ) != 1: 2847 sameLeader = main.FALSE 2848 main.log.error( "Results of electionTestLeader is order of main.CLIs:" + 2849 str( leaders ) ) 2850 utilities.assert_equals( 2851 expect=main.TRUE, 2852 actual=sameLeader, 2853 onpass="Leadership is consistent for the election topic", 2854 onfail="Nodes have different leaders" ) 2855 2856 main.step( "Find current leader and withdraw" ) 2857 leader = main.ONOScli1.electionTestLeader() 2858 # do some sanity checking on leader before using it 2859 withdrawResult = main.FALSE 2860 if leader is None or leader == main.FALSE: 2861 main.log.error( 2862 "Leader for the election app should be an ONOS node," + 2863 "instead got '" + str( leader ) + "'" ) 2864 leaderResult = main.FALSE 2865 oldLeader = None 2866 for i in range( len( main.CLIs ) ): 2867 if leader == main.nodes[ i ].ip_address: 2868 oldLeader = main.CLIs[ i ] 2869 break 2870 else: # FOR/ELSE statement 2871 main.log.error( "Leader election, could not find current leader" ) 2872 if oldLeader: 2873 withdrawResult = oldLeader.electionTestWithdraw() 2874 utilities.assert_equals( 2875 expect=main.TRUE, 2876 actual=withdrawResult, 2877 onpass="Node was withdrawn from election", 2878 onfail="Node was not withdrawn from election" ) 2879 2880 main.step( "Make sure new leader is elected" ) 2881 # FIXME: use threads 2882 leaderList = [] 2883 for cli in main.CLIs: 2884 leaderN = cli.electionTestLeader() 2885 leaderList.append( leaderN ) 2886 if leaderN == leader: 2887 main.log.error( cli.name + " still sees " + str( leader ) + 2888 " as leader after they withdrew" ) 2889 leaderResult = main.FALSE 2890 elif leaderN == main.FALSE: 2891 # error in response 2892 # TODO: add check for "Command not found:" in the driver, this 2893 # means the app isn't loaded 2894 main.log.error( "Something is wrong with " + 2895 "electionTestLeader function, " + 2896 "check the error logs" ) 2897 leaderResult = main.FALSE 2898 elif leaderN is None: 2899 # node may not have recieved the event yet 2900 time.sleep(7) 2901 leaderN = cli.electionTestLeader() 2902 leaderList.pop() 2903 leaderList.append( leaderN ) 2904 consistentLeader = main.FALSE 2905 if len( set( leaderList ) ) == 1: 2906 main.log.info( "Each Election-app sees '" + 2907 str( leaderList[ 0 ] ) + 2908 "' as the leader" ) 2909 consistentLeader = main.TRUE 2910 else: 2911 main.log.error( 2912 "Inconsistent responses for leader of Election-app:" ) 2913 for n in range( len( leaderList ) ): 2914 main.log.error( "ONOS" + str( n + 1 ) + " response: " + 2915 str( leaderList[ n ] ) ) 2916 leaderResult = leaderResult and consistentLeader 2917 utilities.assert_equals( 2918 expect=main.TRUE, 2919 actual=leaderResult, 2920 onpass="Leadership election passed", 2921 onfail="Something went wrong with Leadership election" ) 2922 2923 main.step( "Run for election on old leader( just so everyone " + 2924 "is in the hat )" ) 2925 if oldLeader: 2926 runResult = oldLeader.electionTestRun() 2927 else: 2928 runResult = main.FALSE 2929 utilities.assert_equals( 2930 expect=main.TRUE, 2931 actual=runResult, 2932 onpass="App re-ran for election", 2933 onfail="App failed to run for election" ) 2934 2935 main.step( "Leader did not change when old leader re-ran" ) 2936 afterRun = main.ONOScli1.electionTestLeader() 2937 # verify leader didn't just change 2938 if afterRun == leaderList[ 0 ]: 2939 afterResult = main.TRUE 2940 else: 2941 afterResult = main.FALSE 2942 2943 utilities.assert_equals( 2944 expect=main.TRUE, 2945 actual=afterResult, 2946 onpass="Old leader successfully re-ran for election", 2947 onfail="Something went wrong with Leadership election after " + 2948 "the old leader re-ran for election" )
2949
2950 - def CASE16( self, main ):
2951 """ 2952 Install Distributed Primitives app 2953 """ 2954 import time 2955 assert main.numCtrls, "main.numCtrls not defined" 2956 assert main, "main not defined" 2957 assert utilities.assert_equals, "utilities.assert_equals not defined" 2958 assert main.CLIs, "main.CLIs not defined" 2959 assert main.nodes, "main.nodes not defined" 2960 2961 # Variables for the distributed primitives tests 2962 global pCounterName 2963 global iCounterName 2964 global pCounterValue 2965 global iCounterValue 2966 global onosSet 2967 global onosSetName 2968 pCounterName = "TestON-Partitions" 2969 iCounterName = "TestON-inMemory" 2970 pCounterValue = 0 2971 iCounterValue = 0 2972 onosSet = set([]) 2973 onosSetName = "TestON-set" 2974 2975 description = "Install Primitives app" 2976 main.case( description ) 2977 main.step( "Install Primitives app" ) 2978 appName = "org.onosproject.distributedprimitives" 2979 appResults = main.CLIs[0].activateApp( appName ) 2980 utilities.assert_equals( expect=main.TRUE, 2981 actual=appResults, 2982 onpass="Primitives app activated", 2983 onfail="Primitives app not activated" ) 2984 time.sleep( 5 ) # To allow all nodes to activate
2985
2986 - def CASE17( self, main ):
2987 """ 2988 Check for basic functionality with distributed primitives 2989 """ 2990 # Make sure variables are defined/set 2991 assert main.numCtrls, "main.numCtrls not defined" 2992 assert main, "main not defined" 2993 assert utilities.assert_equals, "utilities.assert_equals not defined" 2994 assert main.CLIs, "main.CLIs not defined" 2995 assert main.nodes, "main.nodes not defined" 2996 assert pCounterName, "pCounterName not defined" 2997 assert iCounterName, "iCounterName not defined" 2998 assert onosSetName, "onosSetName not defined" 2999 # NOTE: assert fails if value is 0/None/Empty/False 3000 try: 3001 pCounterValue 3002 except NameError: 3003 main.log.error( "pCounterValue not defined, setting to 0" ) 3004 pCounterValue = 0 3005 try: 3006 iCounterValue 3007 except NameError: 3008 main.log.error( "iCounterValue not defined, setting to 0" ) 3009 iCounterValue = 0 3010 try: 3011 onosSet 3012 except NameError: 3013 main.log.error( "onosSet not defined, setting to empty Set" ) 3014 onosSet = set([]) 3015 # Variables for the distributed primitives tests. These are local only 3016 addValue = "a" 3017 addAllValue = "a b c d e f" 3018 retainValue = "c d e f" 3019 3020 description = "Check for basic functionality with distributed " +\ 3021 "primitives" 3022 main.case( description ) 3023 main.caseExplanation = "Test the methods of the distributed " +\ 3024 "primitives (counters and sets) throught the cli" 3025 # DISTRIBUTED ATOMIC COUNTERS 3026 # Partitioned counters 3027 main.step( "Increment then get a default counter on each node" ) 3028 pCounters = [] 3029 threads = [] 3030 addedPValues = [] 3031 for i in range( main.numCtrls ): 3032 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3033 name="counterAddAndGet-" + str( i ), 3034 args=[ pCounterName ] ) 3035 pCounterValue += 1 3036 addedPValues.append( pCounterValue ) 3037 threads.append( t ) 3038 t.start() 3039 3040 for t in threads: 3041 t.join() 3042 pCounters.append( t.result ) 3043 # Check that counter incremented numController times 3044 pCounterResults = True 3045 for i in addedPValues: 3046 tmpResult = i in pCounters 3047 pCounterResults = pCounterResults and tmpResult 3048 if not tmpResult: 3049 main.log.error( str( i ) + " is not in partitioned " 3050 "counter incremented results" ) 3051 utilities.assert_equals( expect=True, 3052 actual=pCounterResults, 3053 onpass="Default counter incremented", 3054 onfail="Error incrementing default" + 3055 " counter" ) 3056 3057 main.step( "Get then Increment a default counter on each node" ) 3058 pCounters = [] 3059 threads = [] 3060 addedPValues = [] 3061 for i in range( main.numCtrls ): 3062 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3063 name="counterGetAndAdd-" + str( i ), 3064 args=[ pCounterName ] ) 3065 addedPValues.append( pCounterValue ) 3066 pCounterValue += 1 3067 threads.append( t ) 3068 t.start() 3069 3070 for t in threads: 3071 t.join() 3072 pCounters.append( t.result ) 3073 # Check that counter incremented numController times 3074 pCounterResults = True 3075 for i in addedPValues: 3076 tmpResult = i in pCounters 3077 pCounterResults = pCounterResults and tmpResult 3078 if not tmpResult: 3079 main.log.error( str( i ) + " is not in partitioned " 3080 "counter incremented results" ) 3081 utilities.assert_equals( expect=True, 3082 actual=pCounterResults, 3083 onpass="Default counter incremented", 3084 onfail="Error incrementing default" + 3085 " counter" ) 3086 3087 main.step( "Counters we added have the correct values" ) 3088 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue ) 3089 utilities.assert_equals( expect=main.TRUE, 3090 actual=incrementCheck, 3091 onpass="Added counters are correct", 3092 onfail="Added counters are incorrect" ) 3093 3094 main.step( "Add -8 to then get a default counter on each node" ) 3095 pCounters = [] 3096 threads = [] 3097 addedPValues = [] 3098 for i in range( main.numCtrls ): 3099 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3100 name="counterIncrement-" + str( i ), 3101 args=[ pCounterName ], 3102 kwargs={ "delta": -8 } ) 3103 pCounterValue += -8 3104 addedPValues.append( pCounterValue ) 3105 threads.append( t ) 3106 t.start() 3107 3108 for t in threads: 3109 t.join() 3110 pCounters.append( t.result ) 3111 # Check that counter incremented numController times 3112 pCounterResults = True 3113 for i in addedPValues: 3114 tmpResult = i in pCounters 3115 pCounterResults = pCounterResults and tmpResult 3116 if not tmpResult: 3117 main.log.error( str( i ) + " is not in partitioned " 3118 "counter incremented results" ) 3119 utilities.assert_equals( expect=True, 3120 actual=pCounterResults, 3121 onpass="Default counter incremented", 3122 onfail="Error incrementing default" + 3123 " counter" ) 3124 3125 main.step( "Add 5 to then get a default counter on each node" ) 3126 pCounters = [] 3127 threads = [] 3128 addedPValues = [] 3129 for i in range( main.numCtrls ): 3130 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3131 name="counterIncrement-" + str( i ), 3132 args=[ pCounterName ], 3133 kwargs={ "delta": 5 } ) 3134 pCounterValue += 5 3135 addedPValues.append( pCounterValue ) 3136 threads.append( t ) 3137 t.start() 3138 3139 for t in threads: 3140 t.join() 3141 pCounters.append( t.result ) 3142 # Check that counter incremented numController times 3143 pCounterResults = True 3144 for i in addedPValues: 3145 tmpResult = i in pCounters 3146 pCounterResults = pCounterResults and tmpResult 3147 if not tmpResult: 3148 main.log.error( str( i ) + " is not in partitioned " 3149 "counter incremented results" ) 3150 utilities.assert_equals( expect=True, 3151 actual=pCounterResults, 3152 onpass="Default counter incremented", 3153 onfail="Error incrementing default" + 3154 " counter" ) 3155 3156 main.step( "Get then add 5 to a default counter on each node" ) 3157 pCounters = [] 3158 threads = [] 3159 addedPValues = [] 3160 for i in range( main.numCtrls ): 3161 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3162 name="counterIncrement-" + str( i ), 3163 args=[ pCounterName ], 3164 kwargs={ "delta": 5 } ) 3165 addedPValues.append( pCounterValue ) 3166 pCounterValue += 5 3167 threads.append( t ) 3168 t.start() 3169 3170 for t in threads: 3171 t.join() 3172 pCounters.append( t.result ) 3173 # Check that counter incremented numController times 3174 pCounterResults = True 3175 for i in addedPValues: 3176 tmpResult = i in pCounters 3177 pCounterResults = pCounterResults and tmpResult 3178 if not tmpResult: 3179 main.log.error( str( i ) + " is not in partitioned " 3180 "counter incremented results" ) 3181 utilities.assert_equals( expect=True, 3182 actual=pCounterResults, 3183 onpass="Default counter incremented", 3184 onfail="Error incrementing default" + 3185 " counter" ) 3186 3187 main.step( "Counters we added have the correct values" ) 3188 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue ) 3189 utilities.assert_equals( expect=main.TRUE, 3190 actual=incrementCheck, 3191 onpass="Added counters are correct", 3192 onfail="Added counters are incorrect" ) 3193 3194 # In-Memory counters 3195 main.step( "Increment and get an in-memory counter on each node" ) 3196 iCounters = [] 3197 addedIValues = [] 3198 threads = [] 3199 for i in range( main.numCtrls ): 3200 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3201 name="icounterIncrement-" + str( i ), 3202 args=[ iCounterName ], 3203 kwargs={ "inMemory": True } ) 3204 iCounterValue += 1 3205 addedIValues.append( iCounterValue ) 3206 threads.append( t ) 3207 t.start() 3208 3209 for t in threads: 3210 t.join() 3211 iCounters.append( t.result ) 3212 # Check that counter incremented numController times 3213 iCounterResults = True 3214 for i in addedIValues: 3215 tmpResult = i in iCounters 3216 iCounterResults = iCounterResults and tmpResult 3217 if not tmpResult: 3218 main.log.error( str( i ) + " is not in the in-memory " 3219 "counter incremented results" ) 3220 utilities.assert_equals( expect=True, 3221 actual=iCounterResults, 3222 onpass="In-memory counter incremented", 3223 onfail="Error incrementing in-memory" + 3224 " counter" ) 3225 3226 main.step( "Get then Increment a in-memory counter on each node" ) 3227 iCounters = [] 3228 threads = [] 3229 addedIValues = [] 3230 for i in range( main.numCtrls ): 3231 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3232 name="counterGetAndAdd-" + str( i ), 3233 args=[ iCounterName ], 3234 kwargs={ "inMemory": True } ) 3235 addedIValues.append( iCounterValue ) 3236 iCounterValue += 1 3237 threads.append( t ) 3238 t.start() 3239 3240 for t in threads: 3241 t.join() 3242 iCounters.append( t.result ) 3243 # Check that counter incremented numController times 3244 iCounterResults = True 3245 for i in addedIValues: 3246 tmpResult = i in iCounters 3247 iCounterResults = iCounterResults and tmpResult 3248 if not tmpResult: 3249 main.log.error( str( i ) + " is not in in-memory " 3250 "counter incremented results" ) 3251 utilities.assert_equals( expect=True, 3252 actual=iCounterResults, 3253 onpass="In-memory counter incremented", 3254 onfail="Error incrementing in-memory" + 3255 " counter" ) 3256 3257 main.step( "Counters we added have the correct values" ) 3258 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 3259 utilities.assert_equals( expect=main.TRUE, 3260 actual=incrementCheck, 3261 onpass="Added counters are correct", 3262 onfail="Added counters are incorrect" ) 3263 3264 main.step( "Add -8 to then get a in-memory counter on each node" ) 3265 iCounters = [] 3266 threads = [] 3267 addedIValues = [] 3268 for i in range( main.numCtrls ): 3269 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3270 name="counterIncrement-" + str( i ), 3271 args=[ iCounterName ], 3272 kwargs={ "delta": -8, "inMemory": True } ) 3273 iCounterValue += -8 3274 addedIValues.append( iCounterValue ) 3275 threads.append( t ) 3276 t.start() 3277 3278 for t in threads: 3279 t.join() 3280 iCounters.append( t.result ) 3281 # Check that counter incremented numController times 3282 iCounterResults = True 3283 for i in addedIValues: 3284 tmpResult = i in iCounters 3285 iCounterResults = iCounterResults and tmpResult 3286 if not tmpResult: 3287 main.log.error( str( i ) + " is not in in-memory " 3288 "counter incremented results" ) 3289 utilities.assert_equals( expect=True, 3290 actual=pCounterResults, 3291 onpass="In-memory counter incremented", 3292 onfail="Error incrementing in-memory" + 3293 " counter" ) 3294 3295 main.step( "Add 5 to then get a in-memory counter on each node" ) 3296 iCounters = [] 3297 threads = [] 3298 addedIValues = [] 3299 for i in range( main.numCtrls ): 3300 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 3301 name="counterIncrement-" + str( i ), 3302 args=[ iCounterName ], 3303 kwargs={ "delta": 5, "inMemory": True } ) 3304 iCounterValue += 5 3305 addedIValues.append( iCounterValue ) 3306 threads.append( t ) 3307 t.start() 3308 3309 for t in threads: 3310 t.join() 3311 iCounters.append( t.result ) 3312 # Check that counter incremented numController times 3313 iCounterResults = True 3314 for i in addedIValues: 3315 tmpResult = i in iCounters 3316 iCounterResults = iCounterResults and tmpResult 3317 if not tmpResult: 3318 main.log.error( str( i ) + " is not in in-memory " 3319 "counter incremented results" ) 3320 utilities.assert_equals( expect=True, 3321 actual=pCounterResults, 3322 onpass="In-memory counter incremented", 3323 onfail="Error incrementing in-memory" + 3324 " counter" ) 3325 3326 main.step( "Get then add 5 to a in-memory counter on each node" ) 3327 iCounters = [] 3328 threads = [] 3329 addedIValues = [] 3330 for i in range( main.numCtrls ): 3331 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 3332 name="counterIncrement-" + str( i ), 3333 args=[ iCounterName ], 3334 kwargs={ "delta": 5, "inMemory": True } ) 3335 addedIValues.append( iCounterValue ) 3336 iCounterValue += 5 3337 threads.append( t ) 3338 t.start() 3339 3340 for t in threads: 3341 t.join() 3342 iCounters.append( t.result ) 3343 # Check that counter incremented numController times 3344 iCounterResults = True 3345 for i in addedIValues: 3346 tmpResult = i in iCounters 3347 iCounterResults = iCounterResults and tmpResult 3348 if not tmpResult: 3349 main.log.error( str( i ) + " is not in in-memory " 3350 "counter incremented results" ) 3351 utilities.assert_equals( expect=True, 3352 actual=iCounterResults, 3353 onpass="In-memory counter incremented", 3354 onfail="Error incrementing in-memory" + 3355 " counter" ) 3356 3357 main.step( "Counters we added have the correct values" ) 3358 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 3359 utilities.assert_equals( expect=main.TRUE, 3360 actual=incrementCheck, 3361 onpass="Added counters are correct", 3362 onfail="Added counters are incorrect" ) 3363 3364 main.step( "Check counters are consistant across nodes" ) 3365 onosCounters = [] 3366 threads = [] 3367 for i in range( main.numCtrls ): 3368 t = main.Thread( target=main.CLIs[i].counters, 3369 name="counters-" + str( i ) ) 3370 threads.append( t ) 3371 t.start() 3372 for t in threads: 3373 t.join() 3374 onosCounters.append( t.result ) 3375 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ] 3376 if all( tmp ): 3377 main.log.info( "Counters are consistent across all nodes" ) 3378 consistentCounterResults = main.TRUE 3379 else: 3380 main.log.error( "Counters are not consistent across all nodes" ) 3381 consistentCounterResults = main.FALSE 3382 utilities.assert_equals( expect=main.TRUE, 3383 actual=consistentCounterResults, 3384 onpass="ONOS counters are consistent " + 3385 "across nodes", 3386 onfail="ONOS Counters are inconsistent " + 3387 "across nodes" ) 3388 3389 main.step( "Counters we added have the correct values" ) 3390 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 3391 incrementCheck = incrementCheck and \ 3392 main.Counters.counterCheck( iCounterName, iCounterValue ) 3393 utilities.assert_equals( expect=main.TRUE, 3394 actual=incrementCheck, 3395 onpass="Added counters are correct", 3396 onfail="Added counters are incorrect" ) 3397 3398 # DISTRIBUTED SETS 3399 main.step( "Distributed Set get" ) 3400 size = len( onosSet ) 3401 getResponses = [] 3402 threads = [] 3403 for i in range( main.numCtrls ): 3404 t = main.Thread( target=main.CLIs[i].setTestGet, 3405 name="setTestGet-" + str( i ), 3406 args=[ onosSetName ] ) 3407 threads.append( t ) 3408 t.start() 3409 for t in threads: 3410 t.join() 3411 getResponses.append( t.result ) 3412 3413 getResults = main.TRUE 3414 for i in range( main.numCtrls ): 3415 if isinstance( getResponses[ i ], list): 3416 current = set( getResponses[ i ] ) 3417 if len( current ) == len( getResponses[ i ] ): 3418 # no repeats 3419 if onosSet != current: 3420 main.log.error( "ONOS" + str( i + 1 ) + 3421 " has incorrect view" + 3422 " of set " + onosSetName + ":\n" + 3423 str( getResponses[ i ] ) ) 3424 main.log.debug( "Expected: " + str( onosSet ) ) 3425 main.log.debug( "Actual: " + str( current ) ) 3426 getResults = main.FALSE 3427 else: 3428 # error, set is not a set 3429 main.log.error( "ONOS" + str( i + 1 ) + 3430 " has repeat elements in" + 3431 " set " + onosSetName + ":\n" + 3432 str( getResponses[ i ] ) ) 3433 getResults = main.FALSE 3434 elif getResponses[ i ] == main.ERROR: 3435 getResults = main.FALSE 3436 utilities.assert_equals( expect=main.TRUE, 3437 actual=getResults, 3438 onpass="Set elements are correct", 3439 onfail="Set elements are incorrect" ) 3440 3441 main.step( "Distributed Set size" ) 3442 sizeResponses = [] 3443 threads = [] 3444 for i in range( main.numCtrls ): 3445 t = main.Thread( target=main.CLIs[i].setTestSize, 3446 name="setTestSize-" + str( i ), 3447 args=[ onosSetName ] ) 3448 threads.append( t ) 3449 t.start() 3450 for t in threads: 3451 t.join() 3452 sizeResponses.append( t.result ) 3453 3454 sizeResults = main.TRUE 3455 for i in range( main.numCtrls ): 3456 if size != sizeResponses[ i ]: 3457 sizeResults = main.FALSE 3458 main.log.error( "ONOS" + str( i + 1 ) + 3459 " expected a size of " + str( size ) + 3460 " for set " + onosSetName + 3461 " but got " + str( sizeResponses[ i ] ) ) 3462 utilities.assert_equals( expect=main.TRUE, 3463 actual=sizeResults, 3464 onpass="Set sizes are correct", 3465 onfail="Set sizes are incorrect" ) 3466 3467 main.step( "Distributed Set add()" ) 3468 onosSet.add( addValue ) 3469 addResponses = [] 3470 threads = [] 3471 for i in range( main.numCtrls ): 3472 t = main.Thread( target=main.CLIs[i].setTestAdd, 3473 name="setTestAdd-" + str( i ), 3474 args=[ onosSetName, addValue ] ) 3475 threads.append( t ) 3476 t.start() 3477 for t in threads: 3478 t.join() 3479 addResponses.append( t.result ) 3480 3481 # main.TRUE = successfully changed the set 3482 # main.FALSE = action resulted in no change in set 3483 # main.ERROR - Some error in executing the function 3484 addResults = main.TRUE 3485 for i in range( main.numCtrls ): 3486 if addResponses[ i ] == main.TRUE: 3487 # All is well 3488 pass 3489 elif addResponses[ i ] == main.FALSE: 3490 # Already in set, probably fine 3491 pass 3492 elif addResponses[ i ] == main.ERROR: 3493 # Error in execution 3494 addResults = main.FALSE 3495 else: 3496 # unexpected result 3497 addResults = main.FALSE 3498 if addResults != main.TRUE: 3499 main.log.error( "Error executing set add" ) 3500 3501 # Check if set is still correct 3502 size = len( onosSet ) 3503 getResponses = [] 3504 threads = [] 3505 for i in range( main.numCtrls ): 3506 t = main.Thread( target=main.CLIs[i].setTestGet, 3507 name="setTestGet-" + str( i ), 3508 args=[ onosSetName ] ) 3509 threads.append( t ) 3510 t.start() 3511 for t in threads: 3512 t.join() 3513 getResponses.append( t.result ) 3514 getResults = main.TRUE 3515 for i in range( main.numCtrls ): 3516 if isinstance( getResponses[ i ], list): 3517 current = set( getResponses[ i ] ) 3518 if len( current ) == len( getResponses[ i ] ): 3519 # no repeats 3520 if onosSet != current: 3521 main.log.error( "ONOS" + str( i + 1 ) + 3522 " has incorrect view" + 3523 " of set " + onosSetName + ":\n" + 3524 str( getResponses[ i ] ) ) 3525 main.log.debug( "Expected: " + str( onosSet ) ) 3526 main.log.debug( "Actual: " + str( current ) ) 3527 getResults = main.FALSE 3528 else: 3529 # error, set is not a set 3530 main.log.error( "ONOS" + str( i + 1 ) + 3531 " has repeat elements in" + 3532 " set " + onosSetName + ":\n" + 3533 str( getResponses[ i ] ) ) 3534 getResults = main.FALSE 3535 elif getResponses[ i ] == main.ERROR: 3536 getResults = main.FALSE 3537 sizeResponses = [] 3538 threads = [] 3539 for i in range( main.numCtrls ): 3540 t = main.Thread( target=main.CLIs[i].setTestSize, 3541 name="setTestSize-" + str( i ), 3542 args=[ onosSetName ] ) 3543 threads.append( t ) 3544 t.start() 3545 for t in threads: 3546 t.join() 3547 sizeResponses.append( t.result ) 3548 sizeResults = main.TRUE 3549 for i in range( main.numCtrls ): 3550 if size != sizeResponses[ i ]: 3551 sizeResults = main.FALSE 3552 main.log.error( "ONOS" + str( i + 1 ) + 3553 " expected a size of " + str( size ) + 3554 " for set " + onosSetName + 3555 " but got " + str( sizeResponses[ i ] ) ) 3556 addResults = addResults and getResults and sizeResults 3557 utilities.assert_equals( expect=main.TRUE, 3558 actual=addResults, 3559 onpass="Set add correct", 3560 onfail="Set add was incorrect" ) 3561 3562 main.step( "Distributed Set addAll()" ) 3563 onosSet.update( addAllValue.split() ) 3564 addResponses = [] 3565 threads = [] 3566 for i in range( main.numCtrls ): 3567 t = main.Thread( target=main.CLIs[i].setTestAdd, 3568 name="setTestAddAll-" + str( i ), 3569 args=[ onosSetName, addAllValue ] ) 3570 threads.append( t ) 3571 t.start() 3572 for t in threads: 3573 t.join() 3574 addResponses.append( t.result ) 3575 3576 # main.TRUE = successfully changed the set 3577 # main.FALSE = action resulted in no change in set 3578 # main.ERROR - Some error in executing the function 3579 addAllResults = main.TRUE 3580 for i in range( main.numCtrls ): 3581 if addResponses[ i ] == main.TRUE: 3582 # All is well 3583 pass 3584 elif addResponses[ i ] == main.FALSE: 3585 # Already in set, probably fine 3586 pass 3587 elif addResponses[ i ] == main.ERROR: 3588 # Error in execution 3589 addAllResults = main.FALSE 3590 else: 3591 # unexpected result 3592 addAllResults = main.FALSE 3593 if addAllResults != main.TRUE: 3594 main.log.error( "Error executing set addAll" ) 3595 3596 # Check if set is still correct 3597 size = len( onosSet ) 3598 getResponses = [] 3599 threads = [] 3600 for i in range( main.numCtrls ): 3601 t = main.Thread( target=main.CLIs[i].setTestGet, 3602 name="setTestGet-" + str( i ), 3603 args=[ onosSetName ] ) 3604 threads.append( t ) 3605 t.start() 3606 for t in threads: 3607 t.join() 3608 getResponses.append( t.result ) 3609 getResults = main.TRUE 3610 for i in range( main.numCtrls ): 3611 if isinstance( getResponses[ i ], list): 3612 current = set( getResponses[ i ] ) 3613 if len( current ) == len( getResponses[ i ] ): 3614 # no repeats 3615 if onosSet != current: 3616 main.log.error( "ONOS" + str( i + 1 ) + 3617 " has incorrect view" + 3618 " of set " + onosSetName + ":\n" + 3619 str( getResponses[ i ] ) ) 3620 main.log.debug( "Expected: " + str( onosSet ) ) 3621 main.log.debug( "Actual: " + str( current ) ) 3622 getResults = main.FALSE 3623 else: 3624 # error, set is not a set 3625 main.log.error( "ONOS" + str( i + 1 ) + 3626 " has repeat elements in" + 3627 " set " + onosSetName + ":\n" + 3628 str( getResponses[ i ] ) ) 3629 getResults = main.FALSE 3630 elif getResponses[ i ] == main.ERROR: 3631 getResults = main.FALSE 3632 sizeResponses = [] 3633 threads = [] 3634 for i in range( main.numCtrls ): 3635 t = main.Thread( target=main.CLIs[i].setTestSize, 3636 name="setTestSize-" + str( i ), 3637 args=[ onosSetName ] ) 3638 threads.append( t ) 3639 t.start() 3640 for t in threads: 3641 t.join() 3642 sizeResponses.append( t.result ) 3643 sizeResults = main.TRUE 3644 for i in range( main.numCtrls ): 3645 if size != sizeResponses[ i ]: 3646 sizeResults = main.FALSE 3647 main.log.error( "ONOS" + str( i + 1 ) + 3648 " expected a size of " + str( size ) + 3649 " for set " + onosSetName + 3650 " but got " + str( sizeResponses[ i ] ) ) 3651 addAllResults = addAllResults and getResults and sizeResults 3652 utilities.assert_equals( expect=main.TRUE, 3653 actual=addAllResults, 3654 onpass="Set addAll correct", 3655 onfail="Set addAll was incorrect" ) 3656 3657 main.step( "Distributed Set contains()" ) 3658 containsResponses = [] 3659 threads = [] 3660 for i in range( main.numCtrls ): 3661 t = main.Thread( target=main.CLIs[i].setTestGet, 3662 name="setContains-" + str( i ), 3663 args=[ onosSetName ], 3664 kwargs={ "values": addValue } ) 3665 threads.append( t ) 3666 t.start() 3667 for t in threads: 3668 t.join() 3669 # NOTE: This is the tuple 3670 containsResponses.append( t.result ) 3671 3672 containsResults = main.TRUE 3673 for i in range( main.numCtrls ): 3674 if containsResponses[ i ] == main.ERROR: 3675 containsResults = main.FALSE 3676 else: 3677 containsResults = containsResults and\ 3678 containsResponses[ i ][ 1 ] 3679 utilities.assert_equals( expect=main.TRUE, 3680 actual=containsResults, 3681 onpass="Set contains is functional", 3682 onfail="Set contains failed" ) 3683 3684 main.step( "Distributed Set containsAll()" ) 3685 containsAllResponses = [] 3686 threads = [] 3687 for i in range( main.numCtrls ): 3688 t = main.Thread( target=main.CLIs[i].setTestGet, 3689 name="setContainsAll-" + str( i ), 3690 args=[ onosSetName ], 3691 kwargs={ "values": addAllValue } ) 3692 threads.append( t ) 3693 t.start() 3694 for t in threads: 3695 t.join() 3696 # NOTE: This is the tuple 3697 containsAllResponses.append( t.result ) 3698 3699 containsAllResults = main.TRUE 3700 for i in range( main.numCtrls ): 3701 if containsResponses[ i ] == main.ERROR: 3702 containsResults = main.FALSE 3703 else: 3704 containsResults = containsResults and\ 3705 containsResponses[ i ][ 1 ] 3706 utilities.assert_equals( expect=main.TRUE, 3707 actual=containsAllResults, 3708 onpass="Set containsAll is functional", 3709 onfail="Set containsAll failed" ) 3710 3711 main.step( "Distributed Set remove()" ) 3712 onosSet.remove( addValue ) 3713 removeResponses = [] 3714 threads = [] 3715 for i in range( main.numCtrls ): 3716 t = main.Thread( target=main.CLIs[i].setTestRemove, 3717 name="setTestRemove-" + str( i ), 3718 args=[ onosSetName, addValue ] ) 3719 threads.append( t ) 3720 t.start() 3721 for t in threads: 3722 t.join() 3723 removeResponses.append( t.result ) 3724 3725 # main.TRUE = successfully changed the set 3726 # main.FALSE = action resulted in no change in set 3727 # main.ERROR - Some error in executing the function 3728 removeResults = main.TRUE 3729 for i in range( main.numCtrls ): 3730 if removeResponses[ i ] == main.TRUE: 3731 # All is well 3732 pass 3733 elif removeResponses[ i ] == main.FALSE: 3734 # not in set, probably fine 3735 pass 3736 elif removeResponses[ i ] == main.ERROR: 3737 # Error in execution 3738 removeResults = main.FALSE 3739 else: 3740 # unexpected result 3741 removeResults = main.FALSE 3742 if removeResults != main.TRUE: 3743 main.log.error( "Error executing set remove" ) 3744 3745 # Check if set is still correct 3746 size = len( onosSet ) 3747 getResponses = [] 3748 threads = [] 3749 for i in range( main.numCtrls ): 3750 t = main.Thread( target=main.CLIs[i].setTestGet, 3751 name="setTestGet-" + str( i ), 3752 args=[ onosSetName ] ) 3753 threads.append( t ) 3754 t.start() 3755 for t in threads: 3756 t.join() 3757 getResponses.append( t.result ) 3758 getResults = main.TRUE 3759 for i in range( main.numCtrls ): 3760 if isinstance( getResponses[ i ], list): 3761 current = set( getResponses[ i ] ) 3762 if len( current ) == len( getResponses[ i ] ): 3763 # no repeats 3764 if onosSet != current: 3765 main.log.error( "ONOS" + str( i + 1 ) + 3766 " has incorrect view" + 3767 " of set " + onosSetName + ":\n" + 3768 str( getResponses[ i ] ) ) 3769 main.log.debug( "Expected: " + str( onosSet ) ) 3770 main.log.debug( "Actual: " + str( current ) ) 3771 getResults = main.FALSE 3772 else: 3773 # error, set is not a set 3774 main.log.error( "ONOS" + str( i + 1 ) + 3775 " has repeat elements in" + 3776 " set " + onosSetName + ":\n" + 3777 str( getResponses[ i ] ) ) 3778 getResults = main.FALSE 3779 elif getResponses[ i ] == main.ERROR: 3780 getResults = main.FALSE 3781 sizeResponses = [] 3782 threads = [] 3783 for i in range( main.numCtrls ): 3784 t = main.Thread( target=main.CLIs[i].setTestSize, 3785 name="setTestSize-" + str( i ), 3786 args=[ onosSetName ] ) 3787 threads.append( t ) 3788 t.start() 3789 for t in threads: 3790 t.join() 3791 sizeResponses.append( t.result ) 3792 sizeResults = main.TRUE 3793 for i in range( main.numCtrls ): 3794 if size != sizeResponses[ i ]: 3795 sizeResults = main.FALSE 3796 main.log.error( "ONOS" + str( i + 1 ) + 3797 " expected a size of " + str( size ) + 3798 " for set " + onosSetName + 3799 " but got " + str( sizeResponses[ i ] ) ) 3800 removeResults = removeResults and getResults and sizeResults 3801 utilities.assert_equals( expect=main.TRUE, 3802 actual=removeResults, 3803 onpass="Set remove correct", 3804 onfail="Set remove was incorrect" ) 3805 3806 main.step( "Distributed Set removeAll()" ) 3807 onosSet.difference_update( addAllValue.split() ) 3808 removeAllResponses = [] 3809 threads = [] 3810 try: 3811 for i in range( main.numCtrls ): 3812 t = main.Thread( target=main.CLIs[i].setTestRemove, 3813 name="setTestRemoveAll-" + str( i ), 3814 args=[ onosSetName, addAllValue ] ) 3815 threads.append( t ) 3816 t.start() 3817 for t in threads: 3818 t.join() 3819 removeAllResponses.append( t.result ) 3820 except Exception, e: 3821 main.log.exception(e) 3822 3823 # main.TRUE = successfully changed the set 3824 # main.FALSE = action resulted in no change in set 3825 # main.ERROR - Some error in executing the function 3826 removeAllResults = main.TRUE 3827 for i in range( main.numCtrls ): 3828 if removeAllResponses[ i ] == main.TRUE: 3829 # All is well 3830 pass 3831 elif removeAllResponses[ i ] == main.FALSE: 3832 # not in set, probably fine 3833 pass 3834 elif removeAllResponses[ i ] == main.ERROR: 3835 # Error in execution 3836 removeAllResults = main.FALSE 3837 else: 3838 # unexpected result 3839 removeAllResults = main.FALSE 3840 if removeAllResults != main.TRUE: 3841 main.log.error( "Error executing set removeAll" ) 3842 3843 # Check if set is still correct 3844 size = len( onosSet ) 3845 getResponses = [] 3846 threads = [] 3847 for i in range( main.numCtrls ): 3848 t = main.Thread( target=main.CLIs[i].setTestGet, 3849 name="setTestGet-" + str( i ), 3850 args=[ onosSetName ] ) 3851 threads.append( t ) 3852 t.start() 3853 for t in threads: 3854 t.join() 3855 getResponses.append( t.result ) 3856 getResults = main.TRUE 3857 for i in range( main.numCtrls ): 3858 if isinstance( getResponses[ i ], list): 3859 current = set( getResponses[ i ] ) 3860 if len( current ) == len( getResponses[ i ] ): 3861 # no repeats 3862 if onosSet != current: 3863 main.log.error( "ONOS" + str( i + 1 ) + 3864 " has incorrect view" + 3865 " of set " + onosSetName + ":\n" + 3866 str( getResponses[ i ] ) ) 3867 main.log.debug( "Expected: " + str( onosSet ) ) 3868 main.log.debug( "Actual: " + str( current ) ) 3869 getResults = main.FALSE 3870 else: 3871 # error, set is not a set 3872 main.log.error( "ONOS" + str( i + 1 ) + 3873 " has repeat elements in" + 3874 " set " + onosSetName + ":\n" + 3875 str( getResponses[ i ] ) ) 3876 getResults = main.FALSE 3877 elif getResponses[ i ] == main.ERROR: 3878 getResults = main.FALSE 3879 sizeResponses = [] 3880 threads = [] 3881 for i in range( main.numCtrls ): 3882 t = main.Thread( target=main.CLIs[i].setTestSize, 3883 name="setTestSize-" + str( i ), 3884 args=[ onosSetName ] ) 3885 threads.append( t ) 3886 t.start() 3887 for t in threads: 3888 t.join() 3889 sizeResponses.append( t.result ) 3890 sizeResults = main.TRUE 3891 for i in range( main.numCtrls ): 3892 if size != sizeResponses[ i ]: 3893 sizeResults = main.FALSE 3894 main.log.error( "ONOS" + str( i + 1 ) + 3895 " expected a size of " + str( size ) + 3896 " for set " + onosSetName + 3897 " but got " + str( sizeResponses[ i ] ) ) 3898 removeAllResults = removeAllResults and getResults and sizeResults 3899 utilities.assert_equals( expect=main.TRUE, 3900 actual=removeAllResults, 3901 onpass="Set removeAll correct", 3902 onfail="Set removeAll was incorrect" ) 3903 3904 main.step( "Distributed Set addAll()" ) 3905 onosSet.update( addAllValue.split() ) 3906 addResponses = [] 3907 threads = [] 3908 for i in range( main.numCtrls ): 3909 t = main.Thread( target=main.CLIs[i].setTestAdd, 3910 name="setTestAddAll-" + str( i ), 3911 args=[ onosSetName, addAllValue ] ) 3912 threads.append( t ) 3913 t.start() 3914 for t in threads: 3915 t.join() 3916 addResponses.append( t.result ) 3917 3918 # main.TRUE = successfully changed the set 3919 # main.FALSE = action resulted in no change in set 3920 # main.ERROR - Some error in executing the function 3921 addAllResults = main.TRUE 3922 for i in range( main.numCtrls ): 3923 if addResponses[ i ] == main.TRUE: 3924 # All is well 3925 pass 3926 elif addResponses[ i ] == main.FALSE: 3927 # Already in set, probably fine 3928 pass 3929 elif addResponses[ i ] == main.ERROR: 3930 # Error in execution 3931 addAllResults = main.FALSE 3932 else: 3933 # unexpected result 3934 addAllResults = main.FALSE 3935 if addAllResults != main.TRUE: 3936 main.log.error( "Error executing set addAll" ) 3937 3938 # Check if set is still correct 3939 size = len( onosSet ) 3940 getResponses = [] 3941 threads = [] 3942 for i in range( main.numCtrls ): 3943 t = main.Thread( target=main.CLIs[i].setTestGet, 3944 name="setTestGet-" + str( i ), 3945 args=[ onosSetName ] ) 3946 threads.append( t ) 3947 t.start() 3948 for t in threads: 3949 t.join() 3950 getResponses.append( t.result ) 3951 getResults = main.TRUE 3952 for i in range( main.numCtrls ): 3953 if isinstance( getResponses[ i ], list): 3954 current = set( getResponses[ i ] ) 3955 if len( current ) == len( getResponses[ i ] ): 3956 # no repeats 3957 if onosSet != current: 3958 main.log.error( "ONOS" + str( i + 1 ) + 3959 " has incorrect view" + 3960 " of set " + onosSetName + ":\n" + 3961 str( getResponses[ i ] ) ) 3962 main.log.debug( "Expected: " + str( onosSet ) ) 3963 main.log.debug( "Actual: " + str( current ) ) 3964 getResults = main.FALSE 3965 else: 3966 # error, set is not a set 3967 main.log.error( "ONOS" + str( i + 1 ) + 3968 " has repeat elements in" + 3969 " set " + onosSetName + ":\n" + 3970 str( getResponses[ i ] ) ) 3971 getResults = main.FALSE 3972 elif getResponses[ i ] == main.ERROR: 3973 getResults = main.FALSE 3974 sizeResponses = [] 3975 threads = [] 3976 for i in range( main.numCtrls ): 3977 t = main.Thread( target=main.CLIs[i].setTestSize, 3978 name="setTestSize-" + str( i ), 3979 args=[ onosSetName ] ) 3980 threads.append( t ) 3981 t.start() 3982 for t in threads: 3983 t.join() 3984 sizeResponses.append( t.result ) 3985 sizeResults = main.TRUE 3986 for i in range( main.numCtrls ): 3987 if size != sizeResponses[ i ]: 3988 sizeResults = main.FALSE 3989 main.log.error( "ONOS" + str( i + 1 ) + 3990 " expected a size of " + str( size ) + 3991 " for set " + onosSetName + 3992 " but got " + str( sizeResponses[ i ] ) ) 3993 addAllResults = addAllResults and getResults and sizeResults 3994 utilities.assert_equals( expect=main.TRUE, 3995 actual=addAllResults, 3996 onpass="Set addAll correct", 3997 onfail="Set addAll was incorrect" ) 3998 3999 main.step( "Distributed Set clear()" ) 4000 onosSet.clear() 4001 clearResponses = [] 4002 threads = [] 4003 for i in range( main.numCtrls ): 4004 t = main.Thread( target=main.CLIs[i].setTestRemove, 4005 name="setTestClear-" + str( i ), 4006 args=[ onosSetName, " "], # Values doesn't matter 4007 kwargs={ "clear": True } ) 4008 threads.append( t ) 4009 t.start() 4010 for t in threads: 4011 t.join() 4012 clearResponses.append( t.result ) 4013 4014 # main.TRUE = successfully changed the set 4015 # main.FALSE = action resulted in no change in set 4016 # main.ERROR - Some error in executing the function 4017 clearResults = main.TRUE 4018 for i in range( main.numCtrls ): 4019 if clearResponses[ i ] == main.TRUE: 4020 # All is well 4021 pass 4022 elif clearResponses[ i ] == main.FALSE: 4023 # Nothing set, probably fine 4024 pass 4025 elif clearResponses[ i ] == main.ERROR: 4026 # Error in execution 4027 clearResults = main.FALSE 4028 else: 4029 # unexpected result 4030 clearResults = main.FALSE 4031 if clearResults != main.TRUE: 4032 main.log.error( "Error executing set clear" ) 4033 4034 # Check if set is still correct 4035 size = len( onosSet ) 4036 getResponses = [] 4037 threads = [] 4038 for i in range( main.numCtrls ): 4039 t = main.Thread( target=main.CLIs[i].setTestGet, 4040 name="setTestGet-" + str( i ), 4041 args=[ onosSetName ] ) 4042 threads.append( t ) 4043 t.start() 4044 for t in threads: 4045 t.join() 4046 getResponses.append( t.result ) 4047 getResults = main.TRUE 4048 for i in range( main.numCtrls ): 4049 if isinstance( getResponses[ i ], list): 4050 current = set( getResponses[ i ] ) 4051 if len( current ) == len( getResponses[ i ] ): 4052 # no repeats 4053 if onosSet != current: 4054 main.log.error( "ONOS" + str( i + 1 ) + 4055 " has incorrect view" + 4056 " of set " + onosSetName + ":\n" + 4057 str( getResponses[ i ] ) ) 4058 main.log.debug( "Expected: " + str( onosSet ) ) 4059 main.log.debug( "Actual: " + str( current ) ) 4060 getResults = main.FALSE 4061 else: 4062 # error, set is not a set 4063 main.log.error( "ONOS" + str( i + 1 ) + 4064 " has repeat elements in" + 4065 " set " + onosSetName + ":\n" + 4066 str( getResponses[ i ] ) ) 4067 getResults = main.FALSE 4068 elif getResponses[ i ] == main.ERROR: 4069 getResults = main.FALSE 4070 sizeResponses = [] 4071 threads = [] 4072 for i in range( main.numCtrls ): 4073 t = main.Thread( target=main.CLIs[i].setTestSize, 4074 name="setTestSize-" + str( i ), 4075 args=[ onosSetName ] ) 4076 threads.append( t ) 4077 t.start() 4078 for t in threads: 4079 t.join() 4080 sizeResponses.append( t.result ) 4081 sizeResults = main.TRUE 4082 for i in range( main.numCtrls ): 4083 if size != sizeResponses[ i ]: 4084 sizeResults = main.FALSE 4085 main.log.error( "ONOS" + str( i + 1 ) + 4086 " expected a size of " + str( size ) + 4087 " for set " + onosSetName + 4088 " but got " + str( sizeResponses[ i ] ) ) 4089 clearResults = clearResults and getResults and sizeResults 4090 utilities.assert_equals( expect=main.TRUE, 4091 actual=clearResults, 4092 onpass="Set clear correct", 4093 onfail="Set clear was incorrect" ) 4094 4095 main.step( "Distributed Set addAll()" ) 4096 onosSet.update( addAllValue.split() ) 4097 addResponses = [] 4098 threads = [] 4099 for i in range( main.numCtrls ): 4100 t = main.Thread( target=main.CLIs[i].setTestAdd, 4101 name="setTestAddAll-" + str( i ), 4102 args=[ onosSetName, addAllValue ] ) 4103 threads.append( t ) 4104 t.start() 4105 for t in threads: 4106 t.join() 4107 addResponses.append( t.result ) 4108 4109 # main.TRUE = successfully changed the set 4110 # main.FALSE = action resulted in no change in set 4111 # main.ERROR - Some error in executing the function 4112 addAllResults = main.TRUE 4113 for i in range( main.numCtrls ): 4114 if addResponses[ i ] == main.TRUE: 4115 # All is well 4116 pass 4117 elif addResponses[ i ] == main.FALSE: 4118 # Already in set, probably fine 4119 pass 4120 elif addResponses[ i ] == main.ERROR: 4121 # Error in execution 4122 addAllResults = main.FALSE 4123 else: 4124 # unexpected result 4125 addAllResults = main.FALSE 4126 if addAllResults != main.TRUE: 4127 main.log.error( "Error executing set addAll" ) 4128 4129 # Check if set is still correct 4130 size = len( onosSet ) 4131 getResponses = [] 4132 threads = [] 4133 for i in range( main.numCtrls ): 4134 t = main.Thread( target=main.CLIs[i].setTestGet, 4135 name="setTestGet-" + str( i ), 4136 args=[ onosSetName ] ) 4137 threads.append( t ) 4138 t.start() 4139 for t in threads: 4140 t.join() 4141 getResponses.append( t.result ) 4142 getResults = main.TRUE 4143 for i in range( main.numCtrls ): 4144 if isinstance( getResponses[ i ], list): 4145 current = set( getResponses[ i ] ) 4146 if len( current ) == len( getResponses[ i ] ): 4147 # no repeats 4148 if onosSet != current: 4149 main.log.error( "ONOS" + str( i + 1 ) + 4150 " has incorrect view" + 4151 " of set " + onosSetName + ":\n" + 4152 str( getResponses[ i ] ) ) 4153 main.log.debug( "Expected: " + str( onosSet ) ) 4154 main.log.debug( "Actual: " + str( current ) ) 4155 getResults = main.FALSE 4156 else: 4157 # error, set is not a set 4158 main.log.error( "ONOS" + str( i + 1 ) + 4159 " has repeat elements in" + 4160 " set " + onosSetName + ":\n" + 4161 str( getResponses[ i ] ) ) 4162 getResults = main.FALSE 4163 elif getResponses[ i ] == main.ERROR: 4164 getResults = main.FALSE 4165 sizeResponses = [] 4166 threads = [] 4167 for i in range( main.numCtrls ): 4168 t = main.Thread( target=main.CLIs[i].setTestSize, 4169 name="setTestSize-" + str( i ), 4170 args=[ onosSetName ] ) 4171 threads.append( t ) 4172 t.start() 4173 for t in threads: 4174 t.join() 4175 sizeResponses.append( t.result ) 4176 sizeResults = main.TRUE 4177 for i in range( main.numCtrls ): 4178 if size != sizeResponses[ i ]: 4179 sizeResults = main.FALSE 4180 main.log.error( "ONOS" + str( i + 1 ) + 4181 " expected a size of " + str( size ) + 4182 " for set " + onosSetName + 4183 " but got " + str( sizeResponses[ i ] ) ) 4184 addAllResults = addAllResults and getResults and sizeResults 4185 utilities.assert_equals( expect=main.TRUE, 4186 actual=addAllResults, 4187 onpass="Set addAll correct", 4188 onfail="Set addAll was incorrect" ) 4189 4190 main.step( "Distributed Set retain()" ) 4191 onosSet.intersection_update( retainValue.split() ) 4192 retainResponses = [] 4193 threads = [] 4194 for i in range( main.numCtrls ): 4195 t = main.Thread( target=main.CLIs[i].setTestRemove, 4196 name="setTestRetain-" + str( i ), 4197 args=[ onosSetName, retainValue ], 4198 kwargs={ "retain": True } ) 4199 threads.append( t ) 4200 t.start() 4201 for t in threads: 4202 t.join() 4203 retainResponses.append( t.result ) 4204 4205 # main.TRUE = successfully changed the set 4206 # main.FALSE = action resulted in no change in set 4207 # main.ERROR - Some error in executing the function 4208 retainResults = main.TRUE 4209 for i in range( main.numCtrls ): 4210 if retainResponses[ i ] == main.TRUE: 4211 # All is well 4212 pass 4213 elif retainResponses[ i ] == main.FALSE: 4214 # Already in set, probably fine 4215 pass 4216 elif retainResponses[ i ] == main.ERROR: 4217 # Error in execution 4218 retainResults = main.FALSE 4219 else: 4220 # unexpected result 4221 retainResults = main.FALSE 4222 if retainResults != main.TRUE: 4223 main.log.error( "Error executing set retain" ) 4224 4225 # Check if set is still correct 4226 size = len( onosSet ) 4227 getResponses = [] 4228 threads = [] 4229 for i in range( main.numCtrls ): 4230 t = main.Thread( target=main.CLIs[i].setTestGet, 4231 name="setTestGet-" + str( i ), 4232 args=[ onosSetName ] ) 4233 threads.append( t ) 4234 t.start() 4235 for t in threads: 4236 t.join() 4237 getResponses.append( t.result ) 4238 getResults = main.TRUE 4239 for i in range( main.numCtrls ): 4240 if isinstance( getResponses[ i ], list): 4241 current = set( getResponses[ i ] ) 4242 if len( current ) == len( getResponses[ i ] ): 4243 # no repeats 4244 if onosSet != current: 4245 main.log.error( "ONOS" + str( i + 1 ) + 4246 " has incorrect view" + 4247 " of set " + onosSetName + ":\n" + 4248 str( getResponses[ i ] ) ) 4249 main.log.debug( "Expected: " + str( onosSet ) ) 4250 main.log.debug( "Actual: " + str( current ) ) 4251 getResults = main.FALSE 4252 else: 4253 # error, set is not a set 4254 main.log.error( "ONOS" + str( i + 1 ) + 4255 " has repeat elements in" + 4256 " set " + onosSetName + ":\n" + 4257 str( getResponses[ i ] ) ) 4258 getResults = main.FALSE 4259 elif getResponses[ i ] == main.ERROR: 4260 getResults = main.FALSE 4261 sizeResponses = [] 4262 threads = [] 4263 for i in range( main.numCtrls ): 4264 t = main.Thread( target=main.CLIs[i].setTestSize, 4265 name="setTestSize-" + str( i ), 4266 args=[ onosSetName ] ) 4267 threads.append( t ) 4268 t.start() 4269 for t in threads: 4270 t.join() 4271 sizeResponses.append( t.result ) 4272 sizeResults = main.TRUE 4273 for i in range( main.numCtrls ): 4274 if size != sizeResponses[ i ]: 4275 sizeResults = main.FALSE 4276 main.log.error( "ONOS" + str( i + 1 ) + 4277 " expected a size of " + 4278 str( size ) + " for set " + onosSetName + 4279 " but got " + str( sizeResponses[ i ] ) ) 4280 retainResults = retainResults and getResults and sizeResults 4281 utilities.assert_equals( expect=main.TRUE, 4282 actual=retainResults, 4283 onpass="Set retain correct", 4284 onfail="Set retain was incorrect" )
4285