Package TestON :: Package tests :: Package HAsingleInstanceRestart :: Module HAsingleInstanceRestart
[hide private]
[frames] | no frames]

Source Code for Module TestON.tests.HAsingleInstanceRestart.HAsingleInstanceRestart

   1  """ 
   2  Description: This test is to determine if a single 
   3      instance ONOS 'cluster' can handle a restart 
   4   
   5  List of test cases: 
   6  CASE1: Compile ONOS and push it to the test machines 
   7  CASE2: Assign devices to controllers 
   8  CASE21: Assign mastership to controllers 
   9  CASE3: Assign intents 
  10  CASE4: Ping across added host intents 
  11  CASE5: Reading state of ONOS 
  12  CASE6: The Failure case. 
  13  CASE7: Check state after control plane failure 
  14  CASE8: Compare topo 
  15  CASE9: Link s3-s28 down 
  16  CASE10: Link s3-s28 up 
  17  CASE11: Switch down 
  18  CASE12: Switch up 
  19  CASE13: Clean up 
  20  CASE14: start election app on all onos nodes 
  21  CASE15: Check that Leadership Election is still functional 
  22  CASE16: Install Distributed Primitives app 
  23  CASE17: Check for basic functionality with distributed primitives 
  24  """ 
  25   
  26   
27 -class HAsingleInstanceRestart:
28
29 - def __init__( self ):
30 self.default = ''
31
32 - def CASE1( self, main ):
33 """ 34 CASE1 is to compile ONOS and push it to the test machines 35 36 Startup sequence: 37 cell <name> 38 onos-verify-cell 39 NOTE: temporary - onos-remove-raft-logs 40 onos-uninstall 41 start mininet 42 git pull 43 mvn clean install 44 onos-package 45 onos-install -f 46 onos-wait-for-start 47 start cli sessions 48 start tcpdump 49 """ 50 import imp 51 main.log.info( "ONOS Single node cluster restart " + 52 "HA test - initialization" ) 53 main.case( "Setting up test environment" ) 54 main.caseExplanation = "Setup the test environment including " +\ 55 "installing ONOS, starting Mininet and ONOS" +\ 56 "cli sessions." 57 # TODO: save all the timers and output them for plotting 58 59 # load some variables from the params file 60 PULLCODE = False 61 if main.params[ 'Git' ] == 'True': 62 PULLCODE = True 63 gitBranch = main.params[ 'branch' ] 64 cellName = main.params[ 'ENV' ][ 'cellName' ] 65 66 main.numCtrls = int( main.params[ 'num_controllers' ] ) 67 if main.ONOSbench.maxNodes: 68 if main.ONOSbench.maxNodes < main.numCtrls: 69 main.numCtrls = int( main.ONOSbench.maxNodes ) 70 71 try: 72 fileName = "Counters" 73 path = main.params[ 'imports' ][ 'path' ] 74 main.Counters = imp.load_source( fileName, 75 path + fileName + ".py" ) 76 except Exception as e: 77 main.log.exception( e ) 78 main.cleanup() 79 main.exit() 80 81 main.CLIs = [] 82 main.nodes = [] 83 ipList = [] 84 for i in range( 1, int( main.ONOSbench.maxNodes ) + 1 ): 85 try: 86 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) ) 87 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) ) 88 ipList.append( main.nodes[ -1 ].ip_address ) 89 except AttributeError: 90 break 91 92 main.step( "Create cell file" ) 93 cellAppString = main.params[ 'ENV' ][ 'appString' ] 94 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName, 95 main.Mininet1.ip_address, 96 cellAppString, ipList ) 97 main.step( "Applying cell variable to environment" ) 98 cellResult = main.ONOSbench.setCell( cellName ) 99 verifyResult = main.ONOSbench.verifyCell() 100 101 # FIXME:this is short term fix 102 main.log.info( "Removing raft logs" ) 103 main.ONOSbench.onosRemoveRaftLogs() 104 105 main.log.info( "Uninstalling ONOS" ) 106 for node in main.nodes: 107 main.ONOSbench.onosUninstall( node.ip_address ) 108 109 # Make sure ONOS is DEAD 110 main.log.info( "Killing any ONOS processes" ) 111 killResults = main.TRUE 112 for node in main.nodes: 113 killed = main.ONOSbench.onosKill( node.ip_address ) 114 killResults = killResults and killed 115 116 cleanInstallResult = main.TRUE 117 gitPullResult = main.TRUE 118 119 main.step( "Starting Mininet" ) 120 # scp topo file to mininet 121 # TODO: move to params? 122 topoName = "obelisk.py" 123 filePath = main.ONOSbench.home + "/tools/test/topos/" 124 main.ONOSbench.copyMininetFile( topoName, filePath, 125 main.Mininet1.user_name, 126 main.Mininet1.ip_address ) 127 mnResult = main.Mininet1.startNet( ) 128 utilities.assert_equals( expect=main.TRUE, actual=mnResult, 129 onpass="Mininet Started", 130 onfail="Error starting Mininet" ) 131 132 main.step( "Git checkout and pull " + gitBranch ) 133 if PULLCODE: 134 main.ONOSbench.gitCheckout( gitBranch ) 135 gitPullResult = main.ONOSbench.gitPull() 136 # values of 1 or 3 are good 137 utilities.assert_lesser( expect=0, actual=gitPullResult, 138 onpass="Git pull successful", 139 onfail="Git pull failed" ) 140 main.ONOSbench.getVersion( report=True ) 141 142 main.step( "Using mvn clean install" ) 143 cleanInstallResult = main.TRUE 144 if PULLCODE and gitPullResult == main.TRUE: 145 cleanInstallResult = main.ONOSbench.cleanInstall() 146 else: 147 main.log.warn( "Did not pull new code so skipping mvn " + 148 "clean install" ) 149 utilities.assert_equals( expect=main.TRUE, 150 actual=cleanInstallResult, 151 onpass="MCI successful", 152 onfail="MCI failed" ) 153 # GRAPHS 154 # NOTE: important params here: 155 # job = name of Jenkins job 156 # Plot Name = Plot-HA, only can be used if multiple plots 157 # index = The number of the graph under plot name 158 job = "HAsingleInstanceRestart" 159 plotName = "Plot-HA" 160 graphs = '<ac:structured-macro ac:name="html">\n' 161 graphs += '<ac:plain-text-body><![CDATA[\n' 162 graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\ 163 '/plot/' + plotName + '/getPlot?index=0' +\ 164 '&width=500&height=300"' +\ 165 'noborder="0" width="500" height="300" scrolling="yes" ' +\ 166 'seamless="seamless"></iframe>\n' 167 graphs += ']]></ac:plain-text-body>\n' 168 graphs += '</ac:structured-macro>\n' 169 main.log.wiki(graphs) 170 171 main.CLIs = [] 172 main.nodes = [] 173 ipList = [] 174 for i in range( 1, main.numCtrls + 1 ): 175 main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) ) 176 main.nodes.append( getattr( main, 'ONOS' + str( i ) ) ) 177 ipList.append( main.nodes[ -1 ].ip_address ) 178 179 main.ONOSbench.createCellFile( main.ONOSbench.ip_address, "SingleHA", 180 main.Mininet1.ip_address, 181 cellAppString, ipList[ 0 ] ) 182 cellResult = main.ONOSbench.setCell( "SingleHA" ) 183 verifyResult = main.ONOSbench.verifyCell() 184 main.step( "Creating ONOS package" ) 185 packageResult = main.ONOSbench.onosPackage() 186 utilities.assert_equals( expect=main.TRUE, actual=packageResult, 187 onpass="ONOS package successful", 188 onfail="ONOS package failed" ) 189 190 main.step( "Installing ONOS package" ) 191 onosInstallResult = main.ONOSbench.onosInstall( 192 options="-f", node=main.nodes[0].ip_address ) 193 utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult, 194 onpass="ONOS install successful", 195 onfail="ONOS install failed" ) 196 197 main.step( "Checking if ONOS is up yet" ) 198 for i in range( 2 ): 199 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address ) 200 if onos1Isup: 201 break 202 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup, 203 onpass="ONOS startup successful", 204 onfail="ONOS startup failed" ) 205 206 main.log.step( "Starting ONOS CLI sessions" ) 207 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address ) 208 utilities.assert_equals( expect=main.TRUE, actual=cliResults, 209 onpass="ONOS cli startup successful", 210 onfail="ONOS cli startup failed" ) 211 212 if main.params[ 'tcpdump' ].lower() == "true": 213 main.step( "Start Packet Capture MN" ) 214 main.Mininet2.startTcpdump( 215 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST ) 216 + "-MN.pcap", 217 intf=main.params[ 'MNtcpdump' ][ 'intf' ], 218 port=main.params[ 'MNtcpdump' ][ 'port' ] ) 219 220 main.step( "App Ids check" ) 221 appCheck = main.ONOScli1.appToIDCheck() 222 if appCheck != main.TRUE: 223 main.log.warn( main.CLIs[0].apps() ) 224 main.log.warn( main.CLIs[0].appIDs() ) 225 utilities.assert_equals( expect=main.TRUE, actual=appCheck, 226 onpass="App Ids seem to be correct", 227 onfail="Something is wrong with app Ids" ) 228 229 if cliResults == main.FALSE: 230 main.log.error( "Failed to start ONOS, stopping test" ) 231 main.cleanup() 232 main.exit()
233
234 - def CASE2( self, main ):
235 """ 236 Assign devices to controllers 237 """ 238 import re 239 assert main.numCtrls, "main.numCtrls not defined" 240 assert main, "main not defined" 241 assert utilities.assert_equals, "utilities.assert_equals not defined" 242 243 main.case( "Assigning devices to controllers" ) 244 main.caseExplanation = "Assign switches to ONOS using 'ovs-vsctl' " +\ 245 "and check that an ONOS node becomes the " +\ 246 "master of the device." 247 main.step( "Assign switches to controllers" ) 248 249 ipList = [] 250 for i in range( main.numCtrls ): 251 ipList.append( main.nodes[ i ].ip_address ) 252 swList = [] 253 for i in range( 1, 29 ): 254 swList.append( "s" + str( i ) ) 255 main.Mininet1.assignSwController( sw=swList, ip=ipList ) 256 257 mastershipCheck = main.TRUE 258 for i in range( 1, 29 ): 259 response = main.Mininet1.getSwController( "s" + str( i ) ) 260 try: 261 main.log.info( str( response ) ) 262 except Exception: 263 main.log.info( repr( response ) ) 264 if re.search( "tcp:" + main.nodes[0].ip_address, response ): 265 mastershipCheck = mastershipCheck and main.TRUE 266 else: 267 mastershipCheck = main.FALSE 268 if mastershipCheck == main.TRUE: 269 main.log.info( "Switch mastership assigned correctly" ) 270 utilities.assert_equals( 271 expect=main.TRUE, 272 actual=mastershipCheck, 273 onpass="Switch mastership assigned correctly", 274 onfail="Switches not assigned correctly to controllers" )
275
276 - def CASE21( self, main ):
277 """ 278 Assign mastership to controllers 279 """ 280 assert main.numCtrls, "main.numCtrls not defined" 281 assert main, "main not defined" 282 assert utilities.assert_equals, "utilities.assert_equals not defined" 283 assert main.CLIs, "main.CLIs not defined" 284 assert main.nodes, "main.nodes not defined" 285 286 main.case( "Assigning Controller roles for switches" ) 287 main.caseExplanation = "Check that ONOS is connected to each " +\ 288 "device. Then manually assign" +\ 289 " mastership to specific ONOS nodes using" +\ 290 " 'device-role'" 291 main.step( "Assign mastership of switches to specific controllers" ) 292 roleCall = main.TRUE 293 roleCheck = main.TRUE 294 try: 295 for i in range( 1, 29 ): # switches 1 through 28 296 ip = main.nodes[ 0 ].ip_address # ONOS1 297 # set up correct variables: 298 if i == 1: 299 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' ) 300 elif i == 2: 301 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' ) 302 elif i == 3: 303 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' ) 304 elif i == 4: 305 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' ) 306 elif i == 5: 307 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' ) 308 elif i == 6: 309 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' ) 310 elif i == 7: 311 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' ) 312 elif i >= 8 and i <= 17: 313 dpid = '3' + str( i ).zfill( 3 ) 314 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' ) 315 elif i >= 18 and i <= 27: 316 dpid = '6' + str( i ).zfill( 3 ) 317 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' ) 318 elif i == 28: 319 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' ) 320 else: 321 main.log.error( "You didn't write an else statement for " + 322 "switch s" + str( i ) ) 323 # Assign switch 324 assert deviceId, "No device id for s" + str( i ) + " in ONOS" 325 # TODO: make this controller dynamic 326 roleCall = roleCall and main.ONOScli1.deviceRole( deviceId, 327 ip ) 328 # Check assignment 329 master = main.ONOScli1.getRole( deviceId ).get( 'master' ) 330 if ip in master: 331 roleCheck = roleCheck and main.TRUE 332 else: 333 roleCheck = roleCheck and main.FALSE 334 main.log.error( "Error, controller " + ip + " is not" + 335 " master " + "of device " + 336 str( deviceId ) + ". Master is " + 337 repr( master ) + "." ) 338 except ( AttributeError, AssertionError ): 339 main.log.exception( "Something is wrong with ONOS device view" ) 340 main.log.info( main.ONOScli1.devices() ) 341 utilities.assert_equals( 342 expect=main.TRUE, 343 actual=roleCall, 344 onpass="Re-assigned switch mastership to designated controller", 345 onfail="Something wrong with deviceRole calls" ) 346 347 main.step( "Check mastership was correctly assigned" ) 348 utilities.assert_equals( 349 expect=main.TRUE, 350 actual=roleCheck, 351 onpass="Switches were successfully reassigned to designated " + 352 "controller", 353 onfail="Switches were not successfully reassigned" )
354
355 - def CASE3( self, main ):
356 """ 357 Assign intents 358 """ 359 import time 360 import json 361 assert main.numCtrls, "main.numCtrls not defined" 362 assert main, "main not defined" 363 assert utilities.assert_equals, "utilities.assert_equals not defined" 364 # NOTE: we must reinstall intents until we have a persistant intent 365 # datastore! 366 main.case( "Adding host Intents" ) 367 main.caseExplanation = "Discover hosts by using pingall then " +\ 368 "assign predetermined host-to-host intents." +\ 369 " After installation, check that the intent" +\ 370 " is distributed to all nodes and the state" +\ 371 " is INSTALLED" 372 373 # install onos-app-fwd 374 main.step( "Install reactive forwarding app" ) 375 installResults = main.CLIs[0].activateApp( "org.onosproject.fwd" ) 376 utilities.assert_equals( expect=main.TRUE, actual=installResults, 377 onpass="Install fwd successful", 378 onfail="Install fwd failed" ) 379 380 main.step( "Check app ids" ) 381 appCheck = main.ONOScli1.appToIDCheck() 382 if appCheck != main.TRUE: 383 main.log.warn( main.CLIs[0].apps() ) 384 main.log.warn( main.CLIs[0].appIDs() ) 385 utilities.assert_equals( expect=main.TRUE, actual=appCheck, 386 onpass="App Ids seem to be correct", 387 onfail="Something is wrong with app Ids" ) 388 389 main.step( "Discovering Hosts( Via pingall for now )" ) 390 # FIXME: Once we have a host discovery mechanism, use that instead 391 # REACTIVE FWD test 392 pingResult = main.FALSE 393 for i in range(2): # Retry if pingall fails first time 394 time1 = time.time() 395 pingResult = main.Mininet1.pingall() 396 if i == 0: 397 utilities.assert_equals( 398 expect=main.TRUE, 399 actual=pingResult, 400 onpass="Reactive Pingall test passed", 401 onfail="Reactive Pingall failed, " + 402 "one or more ping pairs failed" ) 403 time2 = time.time() 404 main.log.info( "Time for pingall: %2f seconds" % 405 ( time2 - time1 ) ) 406 # timeout for fwd flows 407 time.sleep( 11 ) 408 # uninstall onos-app-fwd 409 main.step( "Uninstall reactive forwarding app" ) 410 uninstallResult = main.CLIs[0].deactivateApp( "org.onosproject.fwd" ) 411 utilities.assert_equals( expect=main.TRUE, actual=uninstallResult, 412 onpass="Uninstall fwd successful", 413 onfail="Uninstall fwd failed" ) 414 415 main.step( "Check app ids" ) 416 appCheck2 = main.ONOScli1.appToIDCheck() 417 if appCheck2 != main.TRUE: 418 main.log.warn( main.CLIs[0].apps() ) 419 main.log.warn( main.CLIs[0].appIDs() ) 420 utilities.assert_equals( expect=main.TRUE, actual=appCheck2, 421 onpass="App Ids seem to be correct", 422 onfail="Something is wrong with app Ids" ) 423 424 main.step( "Add host intents via cli" ) 425 intentIds = [] 426 # TODO: move the host numbers to params 427 # Maybe look at all the paths we ping? 428 intentAddResult = True 429 hostResult = main.TRUE 430 for i in range( 8, 18 ): 431 main.log.info( "Adding host intent between h" + str( i ) + 432 " and h" + str( i + 10 ) ) 433 host1 = "00:00:00:00:00:" + \ 434 str( hex( i )[ 2: ] ).zfill( 2 ).upper() 435 host2 = "00:00:00:00:00:" + \ 436 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper() 437 # NOTE: getHost can return None 438 host1Dict = main.ONOScli1.getHost( host1 ) 439 host2Dict = main.ONOScli1.getHost( host2 ) 440 host1Id = None 441 host2Id = None 442 if host1Dict and host2Dict: 443 host1Id = host1Dict.get( 'id', None ) 444 host2Id = host2Dict.get( 'id', None ) 445 if host1Id and host2Id: 446 tmpId = main.ONOScli1.addHostIntent( host1Id, host2Id ) 447 if tmpId: 448 main.log.info( "Added intent with id: " + tmpId ) 449 intentIds.append( tmpId ) 450 else: 451 main.log.error( "addHostIntent returned: " + 452 repr( tmpId ) ) 453 else: 454 main.log.error( "Error, getHost() failed for h" + str( i ) + 455 " and/or h" + str( i + 10 ) ) 456 hosts = main.ONOScli1.hosts() 457 main.log.warn( "Hosts output: " ) 458 try: 459 main.log.warn( json.dumps( json.loads( hosts ), 460 sort_keys=True, 461 indent=4, 462 separators=( ',', ': ' ) ) ) 463 except ( ValueError, TypeError ): 464 main.log.warn( repr( hosts ) ) 465 hostResult = main.FALSE 466 utilities.assert_equals( expect=main.TRUE, actual=hostResult, 467 onpass="Found a host id for each host", 468 onfail="Error looking up host ids" ) 469 470 intentStart = time.time() 471 onosIds = main.ONOScli1.getAllIntentsId() 472 main.log.info( "Submitted intents: " + str( intentIds ) ) 473 main.log.info( "Intents in ONOS: " + str( onosIds ) ) 474 for intent in intentIds: 475 if intent in onosIds: 476 pass # intent submitted is in onos 477 else: 478 intentAddResult = False 479 if intentAddResult: 480 intentStop = time.time() 481 else: 482 intentStop = None 483 # Print the intent states 484 intents = main.ONOScli1.intents() 485 intentStates = [] 486 installedCheck = True 487 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 488 count = 0 489 try: 490 for intent in json.loads( intents ): 491 state = intent.get( 'state', None ) 492 if "INSTALLED" not in state: 493 installedCheck = False 494 intentId = intent.get( 'id', None ) 495 intentStates.append( ( intentId, state ) ) 496 except ( ValueError, TypeError ): 497 main.log.exception( "Error parsing intents" ) 498 # add submitted intents not in the store 499 tmplist = [ i for i, s in intentStates ] 500 missingIntents = False 501 for i in intentIds: 502 if i not in tmplist: 503 intentStates.append( ( i, " - " ) ) 504 missingIntents = True 505 intentStates.sort() 506 for i, s in intentStates: 507 count += 1 508 main.log.info( "%-6s%-15s%-15s" % 509 ( str( count ), str( i ), str( s ) ) ) 510 leaders = main.ONOScli1.leaders() 511 try: 512 missing = False 513 if leaders: 514 parsedLeaders = json.loads( leaders ) 515 main.log.warn( json.dumps( parsedLeaders, 516 sort_keys=True, 517 indent=4, 518 separators=( ',', ': ' ) ) ) 519 # check for all intent partitions 520 topics = [] 521 for i in range( 14 ): 522 topics.append( "intent-partition-" + str( i ) ) 523 main.log.debug( topics ) 524 ONOStopics = [ j['topic'] for j in parsedLeaders ] 525 for topic in topics: 526 if topic not in ONOStopics: 527 main.log.error( "Error: " + topic + 528 " not in leaders" ) 529 missing = True 530 else: 531 main.log.error( "leaders() returned None" ) 532 except ( ValueError, TypeError ): 533 main.log.exception( "Error parsing leaders" ) 534 main.log.error( repr( leaders ) ) 535 # Check all nodes 536 if missing: 537 response = main.ONOScli1.leaders( jsonFormat=False) 538 main.log.warn( "ONOS1 leaders output: \n" + 539 str( response ) ) 540 541 partitions = main.ONOScli1.partitions() 542 try: 543 if partitions : 544 parsedPartitions = json.loads( partitions ) 545 main.log.warn( json.dumps( parsedPartitions, 546 sort_keys=True, 547 indent=4, 548 separators=( ',', ': ' ) ) ) 549 # TODO check for a leader in all paritions 550 # TODO check for consistency among nodes 551 else: 552 main.log.error( "partitions() returned None" ) 553 except ( ValueError, TypeError ): 554 main.log.exception( "Error parsing partitions" ) 555 main.log.error( repr( partitions ) ) 556 pendingMap = main.ONOScli1.pendingMap() 557 try: 558 if pendingMap : 559 parsedPending = json.loads( pendingMap ) 560 main.log.warn( json.dumps( parsedPending, 561 sort_keys=True, 562 indent=4, 563 separators=( ',', ': ' ) ) ) 564 # TODO check something here? 565 else: 566 main.log.error( "pendingMap() returned None" ) 567 except ( ValueError, TypeError ): 568 main.log.exception( "Error parsing pending map" ) 569 main.log.error( repr( pendingMap ) ) 570 571 intentAddResult = bool( intentAddResult and not missingIntents and 572 installedCheck ) 573 if not intentAddResult: 574 main.log.error( "Error in pushing host intents to ONOS" ) 575 576 main.step( "Intent Anti-Entropy dispersion" ) 577 for i in range(100): 578 correct = True 579 main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) ) 580 for cli in main.CLIs: 581 onosIds = [] 582 ids = cli.getAllIntentsId() 583 onosIds.append( ids ) 584 main.log.debug( "Intents in " + cli.name + ": " + 585 str( sorted( onosIds ) ) ) 586 if sorted( ids ) != sorted( intentIds ): 587 main.log.warn( "Set of intent IDs doesn't match" ) 588 correct = False 589 break 590 else: 591 intents = json.loads( cli.intents() ) 592 for intent in intents: 593 if intent[ 'state' ] != "INSTALLED": 594 main.log.warn( "Intent " + intent[ 'id' ] + 595 " is " + intent[ 'state' ] ) 596 correct = False 597 break 598 if correct: 599 break 600 else: 601 time.sleep(1) 602 if not intentStop: 603 intentStop = time.time() 604 global gossipTime 605 gossipTime = intentStop - intentStart 606 main.log.info( "It took about " + str( gossipTime ) + 607 " seconds for all intents to appear in each node" ) 608 # FIXME: make this time configurable/calculate based off of number of 609 # nodes and gossip rounds 610 utilities.assert_greater_equals( 611 expect=40, actual=gossipTime, 612 onpass="ECM anti-entropy for intents worked within " + 613 "expected time", 614 onfail="Intent ECM anti-entropy took too long" ) 615 if gossipTime <= 40: 616 intentAddResult = True 617 618 if not intentAddResult or "key" in pendingMap: 619 import time 620 installedCheck = True 621 main.log.info( "Sleeping 60 seconds to see if intents are found" ) 622 time.sleep( 60 ) 623 onosIds = main.ONOScli1.getAllIntentsId() 624 main.log.info( "Submitted intents: " + str( intentIds ) ) 625 main.log.info( "Intents in ONOS: " + str( onosIds ) ) 626 # Print the intent states 627 intents = main.ONOScli1.intents() 628 intentStates = [] 629 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 630 count = 0 631 try: 632 for intent in json.loads( intents ): 633 # Iter through intents of a node 634 state = intent.get( 'state', None ) 635 if "INSTALLED" not in state: 636 installedCheck = False 637 intentId = intent.get( 'id', None ) 638 intentStates.append( ( intentId, state ) ) 639 except ( ValueError, TypeError ): 640 main.log.exception( "Error parsing intents" ) 641 # add submitted intents not in the store 642 tmplist = [ i for i, s in intentStates ] 643 for i in intentIds: 644 if i not in tmplist: 645 intentStates.append( ( i, " - " ) ) 646 intentStates.sort() 647 for i, s in intentStates: 648 count += 1 649 main.log.info( "%-6s%-15s%-15s" % 650 ( str( count ), str( i ), str( s ) ) ) 651 leaders = main.ONOScli1.leaders() 652 try: 653 missing = False 654 if leaders: 655 parsedLeaders = json.loads( leaders ) 656 main.log.warn( json.dumps( parsedLeaders, 657 sort_keys=True, 658 indent=4, 659 separators=( ',', ': ' ) ) ) 660 # check for all intent partitions 661 # check for election 662 topics = [] 663 for i in range( 14 ): 664 topics.append( "intent-partition-" + str( i ) ) 665 # FIXME: this should only be after we start the app 666 topics.append( "org.onosproject.election" ) 667 main.log.debug( topics ) 668 ONOStopics = [ j['topic'] for j in parsedLeaders ] 669 for topic in topics: 670 if topic not in ONOStopics: 671 main.log.error( "Error: " + topic + 672 " not in leaders" ) 673 missing = True 674 else: 675 main.log.error( "leaders() returned None" ) 676 except ( ValueError, TypeError ): 677 main.log.exception( "Error parsing leaders" ) 678 main.log.error( repr( leaders ) ) 679 # Check all nodes 680 if missing: 681 response = main.ONOScli1.leaders( jsonFormat=False) 682 main.log.warn( "ONOS1 leaders output: \n" + 683 str( response ) ) 684 partitions = main.ONOScli1.partitions() 685 try: 686 if partitions : 687 parsedPartitions = json.loads( partitions ) 688 main.log.warn( json.dumps( parsedPartitions, 689 sort_keys=True, 690 indent=4, 691 separators=( ',', ': ' ) ) ) 692 # TODO check for a leader in all paritions 693 # TODO check for consistency among nodes 694 else: 695 main.log.error( "partitions() returned None" ) 696 except ( ValueError, TypeError ): 697 main.log.exception( "Error parsing partitions" ) 698 main.log.error( repr( partitions ) ) 699 pendingMap = main.ONOScli1.pendingMap() 700 try: 701 if pendingMap : 702 parsedPending = json.loads( pendingMap ) 703 main.log.warn( json.dumps( parsedPending, 704 sort_keys=True, 705 indent=4, 706 separators=( ',', ': ' ) ) ) 707 # TODO check something here? 708 else: 709 main.log.error( "pendingMap() returned None" ) 710 except ( ValueError, TypeError ): 711 main.log.exception( "Error parsing pending map" ) 712 main.log.error( repr( pendingMap ) )
713
714 - def CASE4( self, main ):
715 """ 716 Ping across added host intents 717 """ 718 import json 719 import time 720 assert main.numCtrls, "main.numCtrls not defined" 721 assert main, "main not defined" 722 assert utilities.assert_equals, "utilities.assert_equals not defined" 723 main.case( "Verify connectivity by sendind traffic across Intents" ) 724 main.caseExplanation = "Ping across added host intents to check " +\ 725 "functionality and check the state of " +\ 726 "the intent" 727 main.step( "Ping across added host intents" ) 728 PingResult = main.TRUE 729 for i in range( 8, 18 ): 730 ping = main.Mininet1.pingHost( src="h" + str( i ), 731 target="h" + str( i + 10 ) ) 732 PingResult = PingResult and ping 733 if ping == main.FALSE: 734 main.log.warn( "Ping failed between h" + str( i ) + 735 " and h" + str( i + 10 ) ) 736 elif ping == main.TRUE: 737 main.log.info( "Ping test passed!" ) 738 # Don't set PingResult or you'd override failures 739 if PingResult == main.FALSE: 740 main.log.error( 741 "Intents have not been installed correctly, pings failed." ) 742 # TODO: pretty print 743 main.log.warn( "ONOS1 intents: " ) 744 try: 745 tmpIntents = main.ONOScli1.intents() 746 main.log.warn( json.dumps( json.loads( tmpIntents ), 747 sort_keys=True, 748 indent=4, 749 separators=( ',', ': ' ) ) ) 750 except ( ValueError, TypeError ): 751 main.log.warn( repr( tmpIntents ) ) 752 utilities.assert_equals( 753 expect=main.TRUE, 754 actual=PingResult, 755 onpass="Intents have been installed correctly and pings work", 756 onfail="Intents have not been installed correctly, pings failed." ) 757 758 main.step( "Check Intent state" ) 759 installedCheck = True 760 # Print the intent states 761 intents = main.ONOScli1.intents() 762 intentStates = [] 763 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 764 count = 0 765 # Iter through intents of a node 766 try: 767 for intent in json.loads( intents ): 768 state = intent.get( 'state', None ) 769 if "INSTALLED" not in state: 770 installedCheck = False 771 intentId = intent.get( 'id', None ) 772 intentStates.append( ( intentId, state ) ) 773 except ( ValueError, TypeError ): 774 main.log.exception( "Error parsing intents." ) 775 # Print states 776 intentStates.sort() 777 for i, s in intentStates: 778 count += 1 779 main.log.info( "%-6s%-15s%-15s" % 780 ( str( count ), str( i ), str( s ) ) ) 781 utilities.assert_equals( expect=True, actual=installedCheck, 782 onpass="Intents are all INSTALLED", 783 onfail="Intents are not all in " + 784 "INSTALLED state" ) 785 786 main.step( "Check leadership of topics" ) 787 leaders = main.ONOScli1.leaders() 788 topicCheck = main.TRUE 789 try: 790 if leaders: 791 parsedLeaders = json.loads( leaders ) 792 main.log.warn( json.dumps( parsedLeaders, 793 sort_keys=True, 794 indent=4, 795 separators=( ',', ': ' ) ) ) 796 # check for all intent partitions 797 # check for election 798 # TODO: Look at Devices as topics now that it uses this system 799 topics = [] 800 for i in range( 14 ): 801 topics.append( "intent-partition-" + str( i ) ) 802 # FIXME: this should only be after we start the app 803 # FIXME: topics.append( "org.onosproject.election" ) 804 # Print leaders output 805 main.log.debug( topics ) 806 ONOStopics = [ j['topic'] for j in parsedLeaders ] 807 for topic in topics: 808 if topic not in ONOStopics: 809 main.log.error( "Error: " + topic + 810 " not in leaders" ) 811 topicCheck = main.FALSE 812 else: 813 main.log.error( "leaders() returned None" ) 814 topicCheck = main.FALSE 815 except ( ValueError, TypeError ): 816 topicCheck = main.FALSE 817 main.log.exception( "Error parsing leaders" ) 818 main.log.error( repr( leaders ) ) 819 # TODO: Check for a leader of these topics 820 utilities.assert_equals( expect=main.TRUE, actual=topicCheck, 821 onpass="intent Partitions is in leaders", 822 onfail="Some topics were lost " ) 823 # Print partitions 824 partitions = main.ONOScli1.partitions() 825 try: 826 if partitions : 827 parsedPartitions = json.loads( partitions ) 828 main.log.warn( json.dumps( parsedPartitions, 829 sort_keys=True, 830 indent=4, 831 separators=( ',', ': ' ) ) ) 832 # TODO check for a leader in all paritions 833 # TODO check for consistency among nodes 834 else: 835 main.log.error( "partitions() returned None" ) 836 except ( ValueError, TypeError ): 837 main.log.exception( "Error parsing partitions" ) 838 main.log.error( repr( partitions ) ) 839 # Print Pending Map 840 pendingMap = main.ONOScli1.pendingMap() 841 try: 842 if pendingMap : 843 parsedPending = json.loads( pendingMap ) 844 main.log.warn( json.dumps( parsedPending, 845 sort_keys=True, 846 indent=4, 847 separators=( ',', ': ' ) ) ) 848 # TODO check something here? 849 else: 850 main.log.error( "pendingMap() returned None" ) 851 except ( ValueError, TypeError ): 852 main.log.exception( "Error parsing pending map" ) 853 main.log.error( repr( pendingMap ) ) 854 855 if not installedCheck: 856 main.log.info( "Waiting 60 seconds to see if the state of " + 857 "intents change" ) 858 time.sleep( 60 ) 859 # Print the intent states 860 intents = main.ONOScli1.intents() 861 intentStates = [] 862 main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) ) 863 count = 0 864 # Iter through intents of a node 865 try: 866 for intent in json.loads( intents ): 867 state = intent.get( 'state', None ) 868 if "INSTALLED" not in state: 869 installedCheck = False 870 intentId = intent.get( 'id', None ) 871 intentStates.append( ( intentId, state ) ) 872 except ( ValueError, TypeError ): 873 main.log.exception( "Error parsing intents." ) 874 intentStates.sort() 875 for i, s in intentStates: 876 count += 1 877 main.log.info( "%-6s%-15s%-15s" % 878 ( str( count ), str( i ), str( s ) ) ) 879 leaders = main.ONOScli1.leaders() 880 try: 881 missing = False 882 if leaders: 883 parsedLeaders = json.loads( leaders ) 884 main.log.warn( json.dumps( parsedLeaders, 885 sort_keys=True, 886 indent=4, 887 separators=( ',', ': ' ) ) ) 888 # check for all intent partitions 889 # check for election 890 topics = [] 891 for i in range( 14 ): 892 topics.append( "intent-partition-" + str( i ) ) 893 # FIXME: this should only be after we start the app 894 topics.append( "org.onosproject.election" ) 895 main.log.debug( topics ) 896 ONOStopics = [ j['topic'] for j in parsedLeaders ] 897 for topic in topics: 898 if topic not in ONOStopics: 899 main.log.error( "Error: " + topic + 900 " not in leaders" ) 901 missing = True 902 else: 903 main.log.error( "leaders() returned None" ) 904 except ( ValueError, TypeError ): 905 main.log.exception( "Error parsing leaders" ) 906 main.log.error( repr( leaders ) ) 907 if missing: 908 response = main.ONOScli1.leaders( jsonFormat=False) 909 main.log.warn( "ONOS1 leaders output: \n" + 910 str( response ) ) 911 partitions = main.ONOScli1.partitions() 912 try: 913 if partitions : 914 parsedPartitions = json.loads( partitions ) 915 main.log.warn( json.dumps( parsedPartitions, 916 sort_keys=True, 917 indent=4, 918 separators=( ',', ': ' ) ) ) 919 # TODO check for a leader in all paritions 920 # TODO check for consistency among nodes 921 else: 922 main.log.error( "partitions() returned None" ) 923 except ( ValueError, TypeError ): 924 main.log.exception( "Error parsing partitions" ) 925 main.log.error( repr( partitions ) ) 926 pendingMap = main.ONOScli1.pendingMap() 927 try: 928 if pendingMap : 929 parsedPending = json.loads( pendingMap ) 930 main.log.warn( json.dumps( parsedPending, 931 sort_keys=True, 932 indent=4, 933 separators=( ',', ': ' ) ) ) 934 # TODO check something here? 935 else: 936 main.log.error( "pendingMap() returned None" ) 937 except ( ValueError, TypeError ): 938 main.log.exception( "Error parsing pending map" ) 939 main.log.error( repr( pendingMap ) ) 940 # Print flowrules 941 main.log.debug( main.CLIs[0].flows( jsonFormat=False ) ) 942 main.step( "Wait a minute then ping again" ) 943 # the wait is above 944 PingResult = main.TRUE 945 for i in range( 8, 18 ): 946 ping = main.Mininet1.pingHost( src="h" + str( i ), 947 target="h" + str( i + 10 ) ) 948 PingResult = PingResult and ping 949 if ping == main.FALSE: 950 main.log.warn( "Ping failed between h" + str( i ) + 951 " and h" + str( i + 10 ) ) 952 elif ping == main.TRUE: 953 main.log.info( "Ping test passed!" ) 954 # Don't set PingResult or you'd override failures 955 if PingResult == main.FALSE: 956 main.log.error( 957 "Intents have not been installed correctly, pings failed." ) 958 # TODO: pretty print 959 main.log.warn( "ONOS1 intents: " ) 960 try: 961 tmpIntents = main.ONOScli1.intents() 962 main.log.warn( json.dumps( json.loads( tmpIntents ), 963 sort_keys=True, 964 indent=4, 965 separators=( ',', ': ' ) ) ) 966 except ( ValueError, TypeError ): 967 main.log.warn( repr( tmpIntents ) ) 968 utilities.assert_equals( 969 expect=main.TRUE, 970 actual=PingResult, 971 onpass="Intents have been installed correctly and pings work", 972 onfail="Intents have not been installed correctly, pings failed." )
973
974 - def CASE5( self, main ):
975 """ 976 Reading state of ONOS 977 """ 978 import json 979 assert main.numCtrls, "main.numCtrls not defined" 980 assert main, "main not defined" 981 assert utilities.assert_equals, "utilities.assert_equals not defined" 982 983 main.case( "Setting up and gathering data for current state" ) 984 # The general idea for this test case is to pull the state of 985 # ( intents,flows, topology,... ) from each ONOS node 986 # We can then compare them with each other and also with past states 987 988 main.step( "Check that each switch has a master" ) 989 global mastershipState 990 mastershipState = '[]' 991 992 # Assert that each device has a master 993 rolesNotNull = main.ONOScli1.rolesNotNull() 994 utilities.assert_equals( 995 expect=main.TRUE, 996 actual=rolesNotNull, 997 onpass="Each device has a master", 998 onfail="Some devices don't have a master assigned" ) 999 1000 main.step( "Get the Mastership of each switch" ) 1001 ONOS1Mastership = main.ONOScli1.roles() 1002 # TODO: Make this a meaningful check 1003 if "Error" in ONOS1Mastership or not ONOS1Mastership: 1004 main.log.error( "Error in getting ONOS roles" ) 1005 main.log.warn( 1006 "ONOS1 mastership response: " + 1007 repr( ONOS1Mastership ) ) 1008 consistentMastership = main.FALSE 1009 else: 1010 mastershipState = ONOS1Mastership 1011 consistentMastership = main.TRUE 1012 1013 main.step( "Get the intents from each controller" ) 1014 global intentState 1015 intentState = [] 1016 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True ) 1017 intentCheck = main.FALSE 1018 if "Error" in ONOS1Intents or not ONOS1Intents: 1019 main.log.error( "Error in getting ONOS intents" ) 1020 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) ) 1021 else: 1022 intentCheck = main.TRUE 1023 1024 main.step( "Get the flows from each controller" ) 1025 global flowState 1026 flowState = [] 1027 flowCheck = main.FALSE 1028 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True ) 1029 if "Error" in ONOS1Flows or not ONOS1Flows: 1030 main.log.error( "Error in getting ONOS flows" ) 1031 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows ) 1032 else: 1033 # TODO: Do a better check, maybe compare flows on switches? 1034 flowState = ONOS1Flows 1035 flowCheck = main.TRUE 1036 1037 main.step( "Get the OF Table entries" ) 1038 global flows 1039 flows = [] 1040 for i in range( 1, 29 ): 1041 flows.append( main.Mininet1.getFlowTable( 1.3, "s" + str( i ) ) ) 1042 if flowCheck == main.FALSE: 1043 for table in flows: 1044 main.log.warn( table ) 1045 # TODO: Compare switch flow tables with ONOS flow tables 1046 1047 main.step( "Collecting topology information from ONOS" ) 1048 devices = [] 1049 devices.append( main.ONOScli1.devices() ) 1050 hosts = [] 1051 hosts.append( json.loads( main.ONOScli1.hosts() ) ) 1052 ports = [] 1053 ports.append( main.ONOScli1.ports() ) 1054 links = [] 1055 links.append( main.ONOScli1.links() ) 1056 clusters = [] 1057 clusters.append( main.ONOScli1.clusters() ) 1058 1059 main.step( "Each host has an IP address" ) 1060 ipResult = main.TRUE 1061 for controller in range( 0, len( hosts ) ): 1062 controllerStr = str( controller + 1 ) 1063 for host in hosts[ controller ]: 1064 if host is None or host.get( 'ipAddresses', [] ) == []: 1065 main.log.error( 1066 "DEBUG:Error with host ips on controller" + 1067 controllerStr + ": " + str( host ) ) 1068 ipResult = main.FALSE 1069 utilities.assert_equals( 1070 expect=main.TRUE, 1071 actual=ipResult, 1072 onpass="The ips of the hosts aren't empty", 1073 onfail="The ip of at least one host is missing" ) 1074 1075 # there should always only be one cluster 1076 main.step( "There is only one dataplane cluster" ) 1077 try: 1078 numClusters = len( json.loads( clusters[ 0 ] ) ) 1079 except ( ValueError, TypeError ): 1080 main.log.exception( "Error parsing clusters[0]: " + 1081 repr( clusters[ 0 ] ) ) 1082 clusterResults = main.FALSE 1083 if numClusters == 1: 1084 clusterResults = main.TRUE 1085 utilities.assert_equals( 1086 expect=1, 1087 actual=numClusters, 1088 onpass="ONOS shows 1 SCC", 1089 onfail="ONOS shows " + str( numClusters ) + " SCCs" ) 1090 1091 main.step( "Comparing ONOS topology to MN" ) 1092 devicesResults = main.TRUE 1093 linksResults = main.TRUE 1094 hostsResults = main.TRUE 1095 mnSwitches = main.Mininet1.getSwitches() 1096 mnLinks = main.Mininet1.getLinks() 1097 mnHosts = main.Mininet1.getHosts() 1098 for controller in range( main.numCtrls ): 1099 controllerStr = str( controller + 1 ) 1100 if devices[ controller ] and ports[ controller ] and\ 1101 "Error" not in devices[ controller ] and\ 1102 "Error" not in ports[ controller ]: 1103 1104 currentDevicesResult = main.Mininet1.compareSwitches( 1105 mnSwitches, 1106 json.loads( devices[ controller ] ), 1107 json.loads( ports[ controller ] ) ) 1108 else: 1109 currentDevicesResult = main.FALSE 1110 utilities.assert_equals( expect=main.TRUE, 1111 actual=currentDevicesResult, 1112 onpass="ONOS" + controllerStr + 1113 " Switches view is correct", 1114 onfail="ONOS" + controllerStr + 1115 " Switches view is incorrect" ) 1116 if links[ controller ] and "Error" not in links[ controller ]: 1117 currentLinksResult = main.Mininet1.compareLinks( 1118 mnSwitches, mnLinks, 1119 json.loads( links[ controller ] ) ) 1120 else: 1121 currentLinksResult = main.FALSE 1122 utilities.assert_equals( expect=main.TRUE, 1123 actual=currentLinksResult, 1124 onpass="ONOS" + controllerStr + 1125 " links view is correct", 1126 onfail="ONOS" + controllerStr + 1127 " links view is incorrect" ) 1128 1129 if hosts[ controller ] or "Error" not in hosts[ controller ]: 1130 currentHostsResult = main.Mininet1.compareHosts( 1131 mnHosts, 1132 hosts[ controller ] ) 1133 else: 1134 currentHostsResult = main.FALSE 1135 utilities.assert_equals( expect=main.TRUE, 1136 actual=currentHostsResult, 1137 onpass="ONOS" + controllerStr + 1138 " hosts exist in Mininet", 1139 onfail="ONOS" + controllerStr + 1140 " hosts don't match Mininet" ) 1141 1142 devicesResults = devicesResults and currentDevicesResult 1143 linksResults = linksResults and currentLinksResult 1144 hostsResults = hostsResults and currentHostsResult 1145 1146 main.step( "Device information is correct" ) 1147 utilities.assert_equals( 1148 expect=main.TRUE, 1149 actual=devicesResults, 1150 onpass="Device information is correct", 1151 onfail="Device information is incorrect" ) 1152 1153 main.step( "Links are correct" ) 1154 utilities.assert_equals( 1155 expect=main.TRUE, 1156 actual=linksResults, 1157 onpass="Link are correct", 1158 onfail="Links are incorrect" ) 1159 1160 main.step( "Hosts are correct" ) 1161 utilities.assert_equals( 1162 expect=main.TRUE, 1163 actual=hostsResults, 1164 onpass="Hosts are correct", 1165 onfail="Hosts are incorrect" )
1166
1167 - def CASE6( self, main ):
1168 """ 1169 The Failure case. 1170 """ 1171 import time 1172 assert main.numCtrls, "main.numCtrls not defined" 1173 assert main, "main not defined" 1174 assert utilities.assert_equals, "utilities.assert_equals not defined" 1175 1176 # Reset non-persistent variables 1177 try: 1178 iCounterValue = 0 1179 except NameError: 1180 main.log.error( "iCounterValue not defined, setting to 0" ) 1181 iCounterValue = 0 1182 1183 main.case( "Restart ONOS node" ) 1184 main.caseExplanation = "Killing ONOS process and restart cli " +\ 1185 "sessions once onos is up." 1186 main.step( "Killing ONOS processes" ) 1187 killResult = main.ONOSbench.onosKill( main.nodes[0].ip_address ) 1188 start = time.time() 1189 utilities.assert_equals( expect=main.TRUE, actual=killResult, 1190 onpass="ONOS Killed", 1191 onfail="Error killing ONOS" ) 1192 1193 main.step( "Checking if ONOS is up yet" ) 1194 count = 0 1195 while count < 10: 1196 onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address ) 1197 if onos1Isup == main.TRUE: 1198 elapsed = time.time() - start 1199 break 1200 else: 1201 count = count + 1 1202 utilities.assert_equals( expect=main.TRUE, actual=onos1Isup, 1203 onpass="ONOS is back up", 1204 onfail="ONOS failed to start" ) 1205 1206 main.log.step( "Starting ONOS CLI sessions" ) 1207 cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address ) 1208 utilities.assert_equals( expect=main.TRUE, actual=cliResults, 1209 onpass="ONOS cli startup successful", 1210 onfail="ONOS cli startup failed" ) 1211 1212 if elapsed: 1213 main.log.info( "ESTIMATE: ONOS took %s seconds to restart" % 1214 str( elapsed ) ) 1215 main.restartTime = elapsed 1216 else: 1217 main.restartTime = -1 1218 time.sleep( 5 ) 1219 # rerun on election apps 1220 main.ONOScli1.electionTestRun()
1221
1222 - def CASE7( self, main ):
1223 """ 1224 Check state after ONOS failure 1225 """ 1226 import json 1227 assert main.numCtrls, "main.numCtrls not defined" 1228 assert main, "main not defined" 1229 assert utilities.assert_equals, "utilities.assert_equals not defined" 1230 main.case( "Running ONOS Constant State Tests" ) 1231 main.step( "Check that each switch has a master" ) 1232 # Assert that each device has a master 1233 rolesNotNull = main.ONOScli1.rolesNotNull() 1234 utilities.assert_equals( 1235 expect=main.TRUE, 1236 actual=rolesNotNull, 1237 onpass="Each device has a master", 1238 onfail="Some devices don't have a master assigned" ) 1239 1240 main.step( "Check if switch roles are consistent across all nodes" ) 1241 ONOS1Mastership = main.ONOScli1.roles() 1242 # FIXME: Refactor this whole case for single instance 1243 if "Error" in ONOS1Mastership or not ONOS1Mastership: 1244 main.log.error( "Error in getting ONOS mastership" ) 1245 main.log.warn( "ONOS1 mastership response: " + 1246 repr( ONOS1Mastership ) ) 1247 consistentMastership = main.FALSE 1248 else: 1249 consistentMastership = main.TRUE 1250 utilities.assert_equals( 1251 expect=main.TRUE, 1252 actual=consistentMastership, 1253 onpass="Switch roles are consistent across all ONOS nodes", 1254 onfail="ONOS nodes have different views of switch roles" ) 1255 1256 description2 = "Compare switch roles from before failure" 1257 main.step( description2 ) 1258 1259 currentJson = json.loads( ONOS1Mastership ) 1260 oldJson = json.loads( mastershipState ) 1261 mastershipCheck = main.TRUE 1262 for i in range( 1, 29 ): 1263 switchDPID = str( 1264 main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) ) 1265 1266 current = [ switch[ 'master' ] for switch in currentJson 1267 if switchDPID in switch[ 'id' ] ] 1268 old = [ switch[ 'master' ] for switch in oldJson 1269 if switchDPID in switch[ 'id' ] ] 1270 if current == old: 1271 mastershipCheck = mastershipCheck and main.TRUE 1272 else: 1273 main.log.warn( "Mastership of switch %s changed" % switchDPID ) 1274 mastershipCheck = main.FALSE 1275 utilities.assert_equals( 1276 expect=main.TRUE, 1277 actual=mastershipCheck, 1278 onpass="Mastership of Switches was not changed", 1279 onfail="Mastership of some switches changed" ) 1280 mastershipCheck = mastershipCheck and consistentMastership 1281 1282 main.step( "Get the intents and compare across all nodes" ) 1283 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True ) 1284 intentCheck = main.FALSE 1285 if "Error" in ONOS1Intents or not ONOS1Intents: 1286 main.log.error( "Error in getting ONOS intents" ) 1287 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) ) 1288 else: 1289 intentCheck = main.TRUE 1290 utilities.assert_equals( 1291 expect=main.TRUE, 1292 actual=intentCheck, 1293 onpass="Intents are consistent across all ONOS nodes", 1294 onfail="ONOS nodes have different views of intents" ) 1295 # Print the intent states 1296 intents = [] 1297 intents.append( ONOS1Intents ) 1298 intentStates = [] 1299 for node in intents: # Iter through ONOS nodes 1300 nodeStates = [] 1301 # Iter through intents of a node 1302 for intent in json.loads( node ): 1303 nodeStates.append( intent[ 'state' ] ) 1304 intentStates.append( nodeStates ) 1305 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ] 1306 main.log.info( dict( out ) ) 1307 1308 # NOTE: Store has no durability, so intents are lost across system 1309 # restarts 1310 """ 1311 main.step( "Compare current intents with intents before the failure" ) 1312 # NOTE: this requires case 5 to pass for intentState to be set. 1313 # maybe we should stop the test if that fails? 1314 sameIntents = main.FALSE 1315 if intentState and intentState == ONOSIntents[ 0 ]: 1316 sameIntents = main.TRUE 1317 main.log.info( "Intents are consistent with before failure" ) 1318 # TODO: possibly the states have changed? we may need to figure out 1319 # what the acceptable states are 1320 elif len( intentState ) == len( ONOSIntents[ 0 ] ): 1321 sameIntents = main.TRUE 1322 try: 1323 before = json.loads( intentState ) 1324 after = json.loads( ONOSIntents[ 0 ] ) 1325 for intent in before: 1326 if intent not in after: 1327 sameIntents = main.FALSE 1328 main.log.debug( "Intent is not currently in ONOS " + 1329 "(at least in the same form):" ) 1330 main.log.debug( json.dumps( intent ) ) 1331 except ( ValueError, TypeError ): 1332 main.log.exception( "Exception printing intents" ) 1333 main.log.debug( repr( ONOSIntents[0] ) ) 1334 main.log.debug( repr( intentState ) ) 1335 if sameIntents == main.FALSE: 1336 try: 1337 main.log.debug( "ONOS intents before: " ) 1338 main.log.debug( json.dumps( json.loads( intentState ), 1339 sort_keys=True, indent=4, 1340 separators=( ',', ': ' ) ) ) 1341 main.log.debug( "Current ONOS intents: " ) 1342 main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ), 1343 sort_keys=True, indent=4, 1344 separators=( ',', ': ' ) ) ) 1345 except ( ValueError, TypeError ): 1346 main.log.exception( "Exception printing intents" ) 1347 main.log.debug( repr( ONOSIntents[0] ) ) 1348 main.log.debug( repr( intentState ) ) 1349 utilities.assert_equals( 1350 expect=main.TRUE, 1351 actual=sameIntents, 1352 onpass="Intents are consistent with before failure", 1353 onfail="The Intents changed during failure" ) 1354 intentCheck = intentCheck and sameIntents 1355 """ 1356 main.step( "Get the OF Table entries and compare to before " + 1357 "component failure" ) 1358 FlowTables = main.TRUE 1359 flows2 = [] 1360 for i in range( 28 ): 1361 main.log.info( "Checking flow table on s" + str( i + 1 ) ) 1362 tmpFlows = main.Mininet1.getFlowTable( 1.3, "s" + str( i + 1 ) ) 1363 flows2.append( tmpFlows ) 1364 tempResult = main.Mininet1.flowComp( 1365 flow1=flows[ i ], 1366 flow2=tmpFlows ) 1367 FlowTables = FlowTables and tempResult 1368 if FlowTables == main.FALSE: 1369 main.log.info( "Differences in flow table for switch: s" + 1370 str( i + 1 ) ) 1371 utilities.assert_equals( 1372 expect=main.TRUE, 1373 actual=FlowTables, 1374 onpass="No changes were found in the flow tables", 1375 onfail="Changes were found in the flow tables" ) 1376 1377 main.step( "Leadership Election is still functional" ) 1378 # Test of LeadershipElection 1379 1380 leader = main.nodes[0].ip_address 1381 leaderResult = main.TRUE 1382 for controller in range( 1, main.numCtrls + 1 ): 1383 # loop through ONOScli handlers 1384 node = getattr( main, ( 'ONOScli' + str( controller ) ) ) 1385 leaderN = node.electionTestLeader() 1386 # verify leader is ONOS1 1387 # NOTE even though we restarted ONOS, it is the only one so onos 1 1388 # must be leader 1389 if leaderN == leader: 1390 # all is well 1391 pass 1392 elif leaderN == main.FALSE: 1393 # error in response 1394 main.log.error( "Something is wrong with " + 1395 "electionTestLeader function, check the" + 1396 " error logs" ) 1397 leaderResult = main.FALSE 1398 elif leader != leaderN: 1399 leaderResult = main.FALSE 1400 main.log.error( "ONOS" + str( controller ) + " sees " + 1401 str( leaderN ) + 1402 " as the leader of the election app. " + 1403 "Leader should be " + str( leader ) ) 1404 utilities.assert_equals( 1405 expect=main.TRUE, 1406 actual=leaderResult, 1407 onpass="Leadership election passed", 1408 onfail="Something went wrong with Leadership election" )
1409
1410 - def CASE8( self, main ):
1411 """ 1412 Compare topo 1413 """ 1414 import json 1415 import time 1416 assert main.numCtrls, "main.numCtrls not defined" 1417 assert main, "main not defined" 1418 assert utilities.assert_equals, "utilities.assert_equals not defined" 1419 1420 main.case( "Compare ONOS Topology view to Mininet topology" ) 1421 main.caseExplanation = "Compare topology objects between Mininet" +\ 1422 " and ONOS" 1423 1424 main.step( "Comparing ONOS topology to MN" ) 1425 devicesResults = main.TRUE 1426 linksResults = main.TRUE 1427 hostsResults = main.TRUE 1428 hostAttachmentResults = True 1429 topoResult = main.FALSE 1430 elapsed = 0 1431 count = 0 1432 main.step( "Collecting topology information from ONOS" ) 1433 startTime = time.time() 1434 # Give time for Gossip to work 1435 while topoResult == main.FALSE and elapsed < 60: 1436 count += 1 1437 cliStart = time.time() 1438 devices = [] 1439 devices.append( main.ONOScli1.devices() ) 1440 hosts = [] 1441 hosts.append( json.loads( main.ONOScli1.hosts() ) ) 1442 ipResult = main.TRUE 1443 for controller in range( 0, len( hosts ) ): 1444 controllerStr = str( controller + 1 ) 1445 for host in hosts[ controller ]: 1446 if host is None or host.get( 'ipAddresses', [] ) == []: 1447 main.log.error( 1448 "DEBUG:Error with host ips on controller" + 1449 controllerStr + ": " + str( host ) ) 1450 ipResult = main.FALSE 1451 ports = [] 1452 ports.append( main.ONOScli1.ports() ) 1453 links = [] 1454 links.append( main.ONOScli1.links() ) 1455 clusters = [] 1456 clusters.append( main.ONOScli1.clusters() ) 1457 1458 elapsed = time.time() - startTime 1459 cliTime = time.time() - cliStart 1460 print "CLI time: " + str( cliTime ) 1461 1462 mnSwitches = main.Mininet1.getSwitches() 1463 mnLinks = main.Mininet1.getLinks() 1464 mnHosts = main.Mininet1.getHosts() 1465 for controller in range( main.numCtrls ): 1466 controllerStr = str( controller + 1 ) 1467 if devices[ controller ] and ports[ controller ] and\ 1468 "Error" not in devices[ controller ] and\ 1469 "Error" not in ports[ controller ]: 1470 1471 currentDevicesResult = main.Mininet1.compareSwitches( 1472 mnSwitches, 1473 json.loads( devices[ controller ] ), 1474 json.loads( ports[ controller ] ) ) 1475 else: 1476 currentDevicesResult = main.FALSE 1477 utilities.assert_equals( expect=main.TRUE, 1478 actual=currentDevicesResult, 1479 onpass="ONOS" + controllerStr + 1480 " Switches view is correct", 1481 onfail="ONOS" + controllerStr + 1482 " Switches view is incorrect" ) 1483 1484 if links[ controller ] and "Error" not in links[ controller ]: 1485 currentLinksResult = main.Mininet1.compareLinks( 1486 mnSwitches, mnLinks, 1487 json.loads( links[ controller ] ) ) 1488 else: 1489 currentLinksResult = main.FALSE 1490 utilities.assert_equals( expect=main.TRUE, 1491 actual=currentLinksResult, 1492 onpass="ONOS" + controllerStr + 1493 " links view is correct", 1494 onfail="ONOS" + controllerStr + 1495 " links view is incorrect" ) 1496 1497 if hosts[ controller ] or "Error" not in hosts[ controller ]: 1498 currentHostsResult = main.Mininet1.compareHosts( 1499 mnHosts, 1500 hosts[ controller ] ) 1501 else: 1502 currentHostsResult = main.FALSE 1503 utilities.assert_equals( expect=main.TRUE, 1504 actual=currentHostsResult, 1505 onpass="ONOS" + controllerStr + 1506 " hosts exist in Mininet", 1507 onfail="ONOS" + controllerStr + 1508 " hosts don't match Mininet" ) 1509 # CHECKING HOST ATTACHMENT POINTS 1510 hostAttachment = True 1511 zeroHosts = False 1512 # FIXME: topo-HA/obelisk specific mappings: 1513 # key is mac and value is dpid 1514 mappings = {} 1515 for i in range( 1, 29 ): # hosts 1 through 28 1516 # set up correct variables: 1517 macId = "00:" * 5 + hex( i ).split( "0x" )[1].upper().zfill(2) 1518 if i == 1: 1519 deviceId = "1000".zfill(16) 1520 elif i == 2: 1521 deviceId = "2000".zfill(16) 1522 elif i == 3: 1523 deviceId = "3000".zfill(16) 1524 elif i == 4: 1525 deviceId = "3004".zfill(16) 1526 elif i == 5: 1527 deviceId = "5000".zfill(16) 1528 elif i == 6: 1529 deviceId = "6000".zfill(16) 1530 elif i == 7: 1531 deviceId = "6007".zfill(16) 1532 elif i >= 8 and i <= 17: 1533 dpid = '3' + str( i ).zfill( 3 ) 1534 deviceId = dpid.zfill(16) 1535 elif i >= 18 and i <= 27: 1536 dpid = '6' + str( i ).zfill( 3 ) 1537 deviceId = dpid.zfill(16) 1538 elif i == 28: 1539 deviceId = "2800".zfill(16) 1540 mappings[ macId ] = deviceId 1541 if hosts[ controller ] or "Error" not in hosts[ controller ]: 1542 if hosts[ controller ] == []: 1543 main.log.warn( "There are no hosts discovered" ) 1544 zeroHosts = True 1545 else: 1546 for host in hosts[ controller ]: 1547 mac = None 1548 location = None 1549 device = None 1550 port = None 1551 try: 1552 mac = host.get( 'mac' ) 1553 assert mac, "mac field could not be found for this host object" 1554 1555 location = host.get( 'location' ) 1556 assert location, "location field could not be found for this host object" 1557 1558 # Trim the protocol identifier off deviceId 1559 device = str( location.get( 'elementId' ) ).split(':')[1] 1560 assert device, "elementId field could not be found for this host location object" 1561 1562 port = location.get( 'port' ) 1563 assert port, "port field could not be found for this host location object" 1564 1565 # Now check if this matches where they should be 1566 if mac and device and port: 1567 if str( port ) != "1": 1568 main.log.error( "The attachment port is incorrect for " + 1569 "host " + str( mac ) + 1570 ". Expected: 1 Actual: " + str( port) ) 1571 hostAttachment = False 1572 if device != mappings[ str( mac ) ]: 1573 main.log.error( "The attachment device is incorrect for " + 1574 "host " + str( mac ) + 1575 ". Expected: " + mappings[ str( mac ) ] + 1576 " Actual: " + device ) 1577 hostAttachment = False 1578 else: 1579 hostAttachment = False 1580 except AssertionError: 1581 main.log.exception( "Json object not as expected" ) 1582 main.log.error( repr( host ) ) 1583 hostAttachment = False 1584 else: 1585 main.log.error( "No hosts json output or \"Error\"" + 1586 " in output. hosts = " + 1587 repr( hosts[ controller ] ) ) 1588 if zeroHosts is False: 1589 hostAttachment = True 1590 1591 devicesResults = devicesResults and currentDevicesResult 1592 linksResults = linksResults and currentLinksResult 1593 hostsResults = hostsResults and currentHostsResult 1594 hostAttachmentResults = hostAttachmentResults and\ 1595 hostAttachment 1596 1597 # "consistent" results don't make sense for single instance 1598 # there should always only be one cluster 1599 numClusters = len( json.loads( clusters[ 0 ] ) ) 1600 clusterResults = main.FALSE 1601 if numClusters == 1: 1602 clusterResults = main.TRUE 1603 utilities.assert_equals( 1604 expect=1, 1605 actual=numClusters, 1606 onpass="ONOS shows 1 SCC", 1607 onfail="ONOS shows " + str( numClusters ) + " SCCs" ) 1608 1609 topoResult = ( devicesResults and linksResults 1610 and hostsResults and ipResult and clusterResults and 1611 hostAttachmentResults ) 1612 1613 topoResult = topoResult and int( count <= 2 ) 1614 note = "note it takes about " + str( int( cliTime ) ) + \ 1615 " seconds for the test to make all the cli calls to fetch " +\ 1616 "the topology from each ONOS instance" 1617 main.log.info( 1618 "Very crass estimate for topology discovery/convergence( " + 1619 str( note ) + " ): " + str( elapsed ) + " seconds, " + 1620 str( count ) + " tries" ) 1621 utilities.assert_equals( expect=main.TRUE, actual=topoResult, 1622 onpass="Topology Check Test successful", 1623 onfail="Topology Check Test NOT successful" )
1624
1625 - def CASE9( self, main ):
1626 """ 1627 Link s3-s28 down 1628 """ 1629 import time 1630 assert main.numCtrls, "main.numCtrls not defined" 1631 assert main, "main not defined" 1632 assert utilities.assert_equals, "utilities.assert_equals not defined" 1633 # NOTE: You should probably run a topology check after this 1634 1635 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] ) 1636 1637 description = "Turn off a link to ensure that Link Discovery " +\ 1638 "is working properly" 1639 main.case( description ) 1640 1641 main.step( "Kill Link between s3 and s28" ) 1642 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" ) 1643 main.log.info( "Waiting " + str( linkSleep ) + 1644 " seconds for link down to be discovered" ) 1645 time.sleep( linkSleep ) 1646 utilities.assert_equals( expect=main.TRUE, actual=LinkDown, 1647 onpass="Link down successful", 1648 onfail="Failed to bring link down" )
1649 # TODO do some sort of check here 1650
1651 - def CASE10( self, main ):
1652 """ 1653 Link s3-s28 up 1654 """ 1655 import time 1656 assert main.numCtrls, "main.numCtrls not defined" 1657 assert main, "main not defined" 1658 assert utilities.assert_equals, "utilities.assert_equals not defined" 1659 # NOTE: You should probably run a topology check after this 1660 1661 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] ) 1662 1663 description = "Restore a link to ensure that Link Discovery is " + \ 1664 "working properly" 1665 main.case( description ) 1666 1667 main.step( "Bring link between s3 and s28 back up" ) 1668 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" ) 1669 main.log.info( "Waiting " + str( linkSleep ) + 1670 " seconds for link up to be discovered" ) 1671 time.sleep( linkSleep ) 1672 utilities.assert_equals( expect=main.TRUE, actual=LinkUp, 1673 onpass="Link up successful", 1674 onfail="Failed to bring link up" )
1675 # TODO do some sort of check here 1676
1677 - def CASE11( self, main ):
1678 """ 1679 Switch Down 1680 """ 1681 # NOTE: You should probably run a topology check after this 1682 import time 1683 assert main.numCtrls, "main.numCtrls not defined" 1684 assert main, "main not defined" 1685 assert utilities.assert_equals, "utilities.assert_equals not defined" 1686 1687 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) 1688 1689 description = "Killing a switch to ensure it is discovered correctly" 1690 main.case( description ) 1691 switch = main.params[ 'kill' ][ 'switch' ] 1692 switchDPID = main.params[ 'kill' ][ 'dpid' ] 1693 1694 # TODO: Make this switch parameterizable 1695 main.step( "Kill " + switch ) 1696 main.log.info( "Deleting " + switch ) 1697 main.Mininet1.delSwitch( switch ) 1698 main.log.info( "Waiting " + str( switchSleep ) + 1699 " seconds for switch down to be discovered" ) 1700 time.sleep( switchSleep ) 1701 device = main.ONOScli1.getDevice( dpid=switchDPID ) 1702 # Peek at the deleted switch 1703 main.log.warn( str( device ) ) 1704 result = main.FALSE 1705 if device and device[ 'available' ] is False: 1706 result = main.TRUE 1707 utilities.assert_equals( expect=main.TRUE, actual=result, 1708 onpass="Kill switch successful", 1709 onfail="Failed to kill switch?" )
1710
1711 - def CASE12( self, main ):
1712 """ 1713 Switch Up 1714 """ 1715 # NOTE: You should probably run a topology check after this 1716 import time 1717 assert main.numCtrls, "main.numCtrls not defined" 1718 assert main, "main not defined" 1719 assert utilities.assert_equals, "utilities.assert_equals not defined" 1720 1721 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] ) 1722 switch = main.params[ 'kill' ][ 'switch' ] 1723 switchDPID = main.params[ 'kill' ][ 'dpid' ] 1724 links = main.params[ 'kill' ][ 'links' ].split() 1725 description = "Adding a switch to ensure it is discovered correctly" 1726 main.case( description ) 1727 1728 main.step( "Add back " + switch ) 1729 main.Mininet1.addSwitch( switch, dpid=switchDPID ) 1730 for peer in links: 1731 main.Mininet1.addLink( switch, peer ) 1732 ipList = [] 1733 for i in range( main.numCtrls ): 1734 ipList.append( main.nodes[ i ].ip_address ) 1735 main.Mininet1.assignSwController( sw=switch, ip=ipList ) 1736 main.log.info( "Waiting " + str( switchSleep ) + 1737 " seconds for switch up to be discovered" ) 1738 time.sleep( switchSleep ) 1739 device = main.ONOScli1.getDevice( dpid=switchDPID ) 1740 # Peek at the deleted switch 1741 main.log.warn( str( device ) ) 1742 result = main.FALSE 1743 if device and device[ 'available' ]: 1744 result = main.TRUE 1745 utilities.assert_equals( expect=main.TRUE, actual=result, 1746 onpass="add switch successful", 1747 onfail="Failed to add switch?" )
1748
1749 - def CASE13( self, main ):
1750 """ 1751 Clean up 1752 """ 1753 import os 1754 import time 1755 assert main.numCtrls, "main.numCtrls not defined" 1756 assert main, "main not defined" 1757 assert utilities.assert_equals, "utilities.assert_equals not defined" 1758 # printing colors to terminal 1759 colors = { 'cyan': '\033[96m', 'purple': '\033[95m', 1760 'blue': '\033[94m', 'green': '\033[92m', 1761 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' } 1762 main.case( "Test Cleanup" ) 1763 main.step( "Killing tcpdumps" ) 1764 main.Mininet2.stopTcpdump() 1765 1766 testname = main.TEST 1767 if main.params[ 'BACKUP' ] == "True": 1768 main.step( "Copying MN pcap and ONOS log files to test station" ) 1769 teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ] 1770 teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ] 1771 # NOTE: MN Pcap file is being saved to ~/packet_captures 1772 # scp this file as MN and TestON aren't necessarily the same vm 1773 # FIXME: scp 1774 # mn files 1775 # TODO: Load these from params 1776 # NOTE: must end in / 1777 logFolder = "/opt/onos/log/" 1778 logFiles = [ "karaf.log", "karaf.log.1" ] 1779 # NOTE: must end in / 1780 dstDir = "~/packet_captures/" 1781 for f in logFiles: 1782 main.ONOSbench.handle.sendline( "scp sdn@" + main.nodes[0].ip_address + ":" + 1783 logFolder + f + " " + 1784 teststationUser + "@" + 1785 teststationIP + ":" + dstDir + 1786 str( testname ) + "-ONOS1-" + f ) 1787 main.ONOSbench.handle.expect( "\$" ) 1788 1789 # std*.log's 1790 # NOTE: must end in / 1791 logFolder = "/opt/onos/var/" 1792 logFiles = [ "stderr.log", "stdout.log" ] 1793 # NOTE: must end in / 1794 dstDir = "~/packet_captures/" 1795 for f in logFiles: 1796 main.ONOSbench.handle.sendline( "scp sdn@" + main.nodes[0].ip_address + ":" + 1797 logFolder + f + " " + 1798 teststationUser + "@" + 1799 teststationIP + ":" + dstDir + 1800 str( testname ) + "-ONOS1-" + f ) 1801 main.ONOSbench.handle.expect( "\$" ) 1802 # sleep so scp can finish 1803 time.sleep( 10 ) 1804 main.step( "Packing and rotating pcap archives" ) 1805 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) ) 1806 1807 main.step( "Stopping Mininet" ) 1808 mnResult = main.Mininet1.stopNet() 1809 utilities.assert_equals( expect=main.TRUE, actual=mnResult, 1810 onpass="Mininet stopped", 1811 onfail="MN cleanup NOT successful" ) 1812 1813 main.step( "Checking ONOS Logs for errors" ) 1814 print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \ 1815 colors[ 'end' ] 1816 print main.ONOSbench.checkLogs( main.nodes[0].ip_address, restart=True ) 1817 1818 try: 1819 timerLog = open( main.logdir + "/Timers.csv", 'w') 1820 # Overwrite with empty line and close 1821 labels = "Gossip Intents, Restart" 1822 data = str( gossipTime ) + ", " + str( main.restartTime ) 1823 timerLog.write( labels + "\n" + data ) 1824 timerLog.close() 1825 except NameError, e: 1826 main.log.exception(e)
1827
1828 - def CASE14( self, main ):
1829 """ 1830 start election app on all onos nodes 1831 """ 1832 assert main.numCtrls, "main.numCtrls not defined" 1833 assert main, "main not defined" 1834 assert utilities.assert_equals, "utilities.assert_equals not defined" 1835 1836 main.case("Start Leadership Election app") 1837 main.step( "Install leadership election app" ) 1838 appResult = main.ONOScli1.activateApp( "org.onosproject.election" ) 1839 utilities.assert_equals( 1840 expect=main.TRUE, 1841 actual=appResult, 1842 onpass="Election app installed", 1843 onfail="Something went wrong with installing Leadership election" ) 1844 1845 main.step( "Run for election on each node" ) 1846 leaderResult = main.ONOScli1.electionTestRun() 1847 # check for leader 1848 leader = main.ONOScli1.electionTestLeader() 1849 # verify leader is ONOS1 1850 if leader == main.nodes[0].ip_address: 1851 # all is well 1852 pass 1853 elif leader is None: 1854 # No leader elected 1855 main.log.error( "No leader was elected" ) 1856 leaderResult = main.FALSE 1857 elif leader == main.FALSE: 1858 # error in response 1859 # TODO: add check for "Command not found:" in the driver, this 1860 # means the app isn't loaded 1861 main.log.error( "Something is wrong with electionTestLeader" + 1862 " function, check the error logs" ) 1863 leaderResult = main.FALSE 1864 else: 1865 # error in response 1866 main.log.error( 1867 "Unexpected response from electionTestLeader function:'" + 1868 str( leader ) + 1869 "'" ) 1870 leaderResult = main.FALSE 1871 utilities.assert_equals( 1872 expect=main.TRUE, 1873 actual=leaderResult, 1874 onpass="Successfully ran for leadership", 1875 onfail="Failed to run for leadership" )
1876
1877 - def CASE15( self, main ):
1878 """ 1879 Check that Leadership Election is still functional 1880 """ 1881 assert main.numCtrls, "main.numCtrls not defined" 1882 assert main, "main not defined" 1883 assert utilities.assert_equals, "utilities.assert_equals not defined" 1884 leaderResult = main.TRUE 1885 description = "Check that Leadership Election is still functional" 1886 main.case( description ) 1887 main.step( "Find current leader and withdraw" ) 1888 leader = main.ONOScli1.electionTestLeader() 1889 # do some sanity checking on leader before using it 1890 withdrawResult = main.FALSE 1891 if leader == main.nodes[0].ip_address: 1892 oldLeader = getattr( main, "ONOScli1" ) 1893 elif leader is None or leader == main.FALSE: 1894 main.log.error( 1895 "Leader for the election app should be an ONOS node," + 1896 "instead got '" + str( leader ) + "'" ) 1897 leaderResult = main.FALSE 1898 oldLeader = None 1899 else: 1900 main.log.error( "Leader election --- why am I HERE?!?") 1901 leaderResult = main.FALSE 1902 oldLeader = None 1903 if oldLeader: 1904 withdrawResult = oldLeader.electionTestWithdraw() 1905 utilities.assert_equals( 1906 expect=main.TRUE, 1907 actual=withdrawResult, 1908 onpass="Node was withdrawn from election", 1909 onfail="Node was not withdrawn from election" ) 1910 1911 main.step( "Make sure new leader is elected" ) 1912 leaderN = main.ONOScli1.electionTestLeader() 1913 if leaderN == leader: 1914 main.log.error( "ONOS still sees " + str( leaderN ) + 1915 " as leader after they withdrew" ) 1916 leaderResult = main.FALSE 1917 elif leaderN == main.FALSE: 1918 # error in response 1919 # TODO: add check for "Command not found:" in the driver, this 1920 # means the app isn't loaded 1921 main.log.error( "Something is wrong with electionTestLeader " + 1922 "function, check the error logs" ) 1923 leaderResult = main.FALSE 1924 elif leaderN is None: 1925 main.log.info( 1926 "There is no leader after the app withdrew from election" ) 1927 leaderResult = main.TRUE 1928 utilities.assert_equals( 1929 expect=main.TRUE, 1930 actual=leaderResult, 1931 onpass="Leadership election passed", 1932 onfail="Something went wrong with Leadership election" ) 1933 1934 main.step( "Run for election on old leader( just so everyone " + 1935 "is in the hat )" ) 1936 if oldLeader: 1937 runResult = oldLeader.electionTestRun() 1938 else: 1939 runResult = main.FALSE 1940 utilities.assert_equals( 1941 expect=main.TRUE, 1942 actual=runResult, 1943 onpass="App re-ran for election", 1944 onfail="App failed to run for election" ) 1945 1946 main.step( "Node became leader when it ran for election" ) 1947 afterRun = main.ONOScli1.electionTestLeader() 1948 # verify leader is ONOS1 1949 if afterRun == main.nodes[0].ip_address: 1950 afterResult = main.TRUE 1951 else: 1952 afterResult = main.FALSE 1953 1954 utilities.assert_equals( 1955 expect=main.TRUE, 1956 actual=afterResult, 1957 onpass="Old leader successfully re-ran for election", 1958 onfail="Something went wrong with Leadership election after " + 1959 "the old leader re-ran for election" )
1960
1961 - def CASE16( self, main ):
1962 """ 1963 Install Distributed Primitives app 1964 """ 1965 assert main.numCtrls, "main.numCtrls not defined" 1966 assert main, "main not defined" 1967 assert utilities.assert_equals, "utilities.assert_equals not defined" 1968 assert main.CLIs, "main.CLIs not defined" 1969 assert main.nodes, "main.nodes not defined" 1970 1971 # Variables for the distributed primitives tests 1972 global pCounterName 1973 global iCounterName 1974 global pCounterValue 1975 global iCounterValue 1976 global onosSet 1977 global onosSetName 1978 pCounterName = "TestON-Partitions" 1979 iCounterName = "TestON-inMemory" 1980 pCounterValue = 0 1981 iCounterValue = 0 1982 onosSet = set([]) 1983 onosSetName = "TestON-set" 1984 1985 description = "Install Primitives app" 1986 main.case( description ) 1987 main.step( "Install Primitives app" ) 1988 appName = "org.onosproject.distributedprimitives" 1989 appResults = main.CLIs[0].activateApp( appName ) 1990 utilities.assert_equals( expect=main.TRUE, 1991 actual=appResults, 1992 onpass="Primitives app activated", 1993 onfail="Primitives app not activated" )
1994
1995 - def CASE17( self, main ):
1996 """ 1997 Check for basic functionality with distributed primitives 1998 """ 1999 # Make sure variables are defined/set 2000 assert main.numCtrls, "main.numCtrls not defined" 2001 assert main, "main not defined" 2002 assert utilities.assert_equals, "utilities.assert_equals not defined" 2003 assert main.CLIs, "main.CLIs not defined" 2004 assert main.nodes, "main.nodes not defined" 2005 assert pCounterName, "pCounterName not defined" 2006 assert iCounterName, "iCounterName not defined" 2007 assert onosSetName, "onosSetName not defined" 2008 # NOTE: assert fails if value is 0/None/Empty/False 2009 try: 2010 pCounterValue 2011 except NameError: 2012 main.log.error( "pCounterValue not defined, setting to 0" ) 2013 pCounterValue = 0 2014 try: 2015 iCounterValue 2016 except NameError: 2017 main.log.error( "iCounterValue not defined, setting to 0" ) 2018 iCounterValue = 0 2019 try: 2020 onosSet 2021 except NameError: 2022 main.log.error( "onosSet not defined, setting to empty Set" ) 2023 onosSet = set([]) 2024 # Variables for the distributed primitives tests. These are local only 2025 addValue = "a" 2026 addAllValue = "a b c d e f" 2027 retainValue = "c d e f" 2028 2029 description = "Check for basic functionality with distributed " +\ 2030 "primitives" 2031 main.case( description ) 2032 main.caseExplanation = "Test the methods of the distributed " +\ 2033 "primitives (counters and sets) throught the cli" 2034 # DISTRIBUTED ATOMIC COUNTERS 2035 # Partitioned counters 2036 main.step( "Increment then get a default counter on each node" ) 2037 pCounters = [] 2038 threads = [] 2039 addedPValues = [] 2040 for i in range( main.numCtrls ): 2041 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 2042 name="counterAddAndGet-" + str( i ), 2043 args=[ pCounterName ] ) 2044 pCounterValue += 1 2045 addedPValues.append( pCounterValue ) 2046 threads.append( t ) 2047 t.start() 2048 2049 for t in threads: 2050 t.join() 2051 pCounters.append( t.result ) 2052 # Check that counter incremented numController times 2053 pCounterResults = True 2054 for i in addedPValues: 2055 tmpResult = i in pCounters 2056 pCounterResults = pCounterResults and tmpResult 2057 if not tmpResult: 2058 main.log.error( str( i ) + " is not in partitioned " 2059 "counter incremented results" ) 2060 utilities.assert_equals( expect=True, 2061 actual=pCounterResults, 2062 onpass="Default counter incremented", 2063 onfail="Error incrementing default" + 2064 " counter" ) 2065 2066 main.step( "Get then Increment a default counter on each node" ) 2067 pCounters = [] 2068 threads = [] 2069 addedPValues = [] 2070 for i in range( main.numCtrls ): 2071 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 2072 name="counterGetAndAdd-" + str( i ), 2073 args=[ pCounterName ] ) 2074 addedPValues.append( pCounterValue ) 2075 pCounterValue += 1 2076 threads.append( t ) 2077 t.start() 2078 2079 for t in threads: 2080 t.join() 2081 pCounters.append( t.result ) 2082 # Check that counter incremented numController times 2083 pCounterResults = True 2084 for i in addedPValues: 2085 tmpResult = i in pCounters 2086 pCounterResults = pCounterResults and tmpResult 2087 if not tmpResult: 2088 main.log.error( str( i ) + " is not in partitioned " 2089 "counter incremented results" ) 2090 utilities.assert_equals( expect=True, 2091 actual=pCounterResults, 2092 onpass="Default counter incremented", 2093 onfail="Error incrementing default" + 2094 " counter" ) 2095 2096 main.step( "Counters we added have the correct values" ) 2097 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue ) 2098 utilities.assert_equals( expect=main.TRUE, 2099 actual=incrementCheck, 2100 onpass="Added counters are correct", 2101 onfail="Added counters are incorrect" ) 2102 2103 main.step( "Add -8 to then get a default counter on each node" ) 2104 pCounters = [] 2105 threads = [] 2106 addedPValues = [] 2107 for i in range( main.numCtrls ): 2108 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 2109 name="counterIncrement-" + str( i ), 2110 args=[ pCounterName ], 2111 kwargs={ "delta": -8 } ) 2112 pCounterValue += -8 2113 addedPValues.append( pCounterValue ) 2114 threads.append( t ) 2115 t.start() 2116 2117 for t in threads: 2118 t.join() 2119 pCounters.append( t.result ) 2120 # Check that counter incremented numController times 2121 pCounterResults = True 2122 for i in addedPValues: 2123 tmpResult = i in pCounters 2124 pCounterResults = pCounterResults and tmpResult 2125 if not tmpResult: 2126 main.log.error( str( i ) + " is not in partitioned " 2127 "counter incremented results" ) 2128 utilities.assert_equals( expect=True, 2129 actual=pCounterResults, 2130 onpass="Default counter incremented", 2131 onfail="Error incrementing default" + 2132 " counter" ) 2133 2134 main.step( "Add 5 to then get a default counter on each node" ) 2135 pCounters = [] 2136 threads = [] 2137 addedPValues = [] 2138 for i in range( main.numCtrls ): 2139 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 2140 name="counterIncrement-" + str( i ), 2141 args=[ pCounterName ], 2142 kwargs={ "delta": 5 } ) 2143 pCounterValue += 5 2144 addedPValues.append( pCounterValue ) 2145 threads.append( t ) 2146 t.start() 2147 2148 for t in threads: 2149 t.join() 2150 pCounters.append( t.result ) 2151 # Check that counter incremented numController times 2152 pCounterResults = True 2153 for i in addedPValues: 2154 tmpResult = i in pCounters 2155 pCounterResults = pCounterResults and tmpResult 2156 if not tmpResult: 2157 main.log.error( str( i ) + " is not in partitioned " 2158 "counter incremented results" ) 2159 utilities.assert_equals( expect=True, 2160 actual=pCounterResults, 2161 onpass="Default counter incremented", 2162 onfail="Error incrementing default" + 2163 " counter" ) 2164 2165 main.step( "Get then add 5 to a default counter on each node" ) 2166 pCounters = [] 2167 threads = [] 2168 addedPValues = [] 2169 for i in range( main.numCtrls ): 2170 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 2171 name="counterIncrement-" + str( i ), 2172 args=[ pCounterName ], 2173 kwargs={ "delta": 5 } ) 2174 addedPValues.append( pCounterValue ) 2175 pCounterValue += 5 2176 threads.append( t ) 2177 t.start() 2178 2179 for t in threads: 2180 t.join() 2181 pCounters.append( t.result ) 2182 # Check that counter incremented numController times 2183 pCounterResults = True 2184 for i in addedPValues: 2185 tmpResult = i in pCounters 2186 pCounterResults = pCounterResults and tmpResult 2187 if not tmpResult: 2188 main.log.error( str( i ) + " is not in partitioned " 2189 "counter incremented results" ) 2190 utilities.assert_equals( expect=True, 2191 actual=pCounterResults, 2192 onpass="Default counter incremented", 2193 onfail="Error incrementing default" + 2194 " counter" ) 2195 2196 main.step( "Counters we added have the correct values" ) 2197 incrementCheck = main.Counters.counterCheck( pCounterName, pCounterValue ) 2198 utilities.assert_equals( expect=main.TRUE, 2199 actual=incrementCheck, 2200 onpass="Added counters are correct", 2201 onfail="Added counters are incorrect" ) 2202 2203 # In-Memory counters 2204 main.step( "Increment and get an in-memory counter on each node" ) 2205 iCounters = [] 2206 addedIValues = [] 2207 threads = [] 2208 for i in range( main.numCtrls ): 2209 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 2210 name="icounterIncrement-" + str( i ), 2211 args=[ iCounterName ], 2212 kwargs={ "inMemory": True } ) 2213 iCounterValue += 1 2214 addedIValues.append( iCounterValue ) 2215 threads.append( t ) 2216 t.start() 2217 2218 for t in threads: 2219 t.join() 2220 iCounters.append( t.result ) 2221 # Check that counter incremented numController times 2222 iCounterResults = True 2223 for i in addedIValues: 2224 tmpResult = i in iCounters 2225 iCounterResults = iCounterResults and tmpResult 2226 if not tmpResult: 2227 main.log.error( str( i ) + " is not in the in-memory " 2228 "counter incremented results" ) 2229 utilities.assert_equals( expect=True, 2230 actual=iCounterResults, 2231 onpass="In-memory counter incremented", 2232 onfail="Error incrementing in-memory" + 2233 " counter" ) 2234 2235 main.step( "Get then Increment a in-memory counter on each node" ) 2236 iCounters = [] 2237 threads = [] 2238 addedIValues = [] 2239 for i in range( main.numCtrls ): 2240 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 2241 name="counterGetAndAdd-" + str( i ), 2242 args=[ iCounterName ], 2243 kwargs={ "inMemory": True } ) 2244 addedIValues.append( iCounterValue ) 2245 iCounterValue += 1 2246 threads.append( t ) 2247 t.start() 2248 2249 for t in threads: 2250 t.join() 2251 iCounters.append( t.result ) 2252 # Check that counter incremented numController times 2253 iCounterResults = True 2254 for i in addedIValues: 2255 tmpResult = i in iCounters 2256 iCounterResults = iCounterResults and tmpResult 2257 if not tmpResult: 2258 main.log.error( str( i ) + " is not in in-memory " 2259 "counter incremented results" ) 2260 utilities.assert_equals( expect=True, 2261 actual=iCounterResults, 2262 onpass="In-memory counter incremented", 2263 onfail="Error incrementing in-memory" + 2264 " counter" ) 2265 2266 main.step( "Counters we added have the correct values" ) 2267 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 2268 utilities.assert_equals( expect=main.TRUE, 2269 actual=incrementCheck, 2270 onpass="Added counters are correct", 2271 onfail="Added counters are incorrect" ) 2272 2273 main.step( "Add -8 to then get a in-memory counter on each node" ) 2274 iCounters = [] 2275 threads = [] 2276 addedIValues = [] 2277 for i in range( main.numCtrls ): 2278 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 2279 name="counterIncrement-" + str( i ), 2280 args=[ iCounterName ], 2281 kwargs={ "delta": -8, "inMemory": True } ) 2282 iCounterValue += -8 2283 addedIValues.append( iCounterValue ) 2284 threads.append( t ) 2285 t.start() 2286 2287 for t in threads: 2288 t.join() 2289 iCounters.append( t.result ) 2290 # Check that counter incremented numController times 2291 iCounterResults = True 2292 for i in addedIValues: 2293 tmpResult = i in iCounters 2294 iCounterResults = iCounterResults and tmpResult 2295 if not tmpResult: 2296 main.log.error( str( i ) + " is not in in-memory " 2297 "counter incremented results" ) 2298 utilities.assert_equals( expect=True, 2299 actual=pCounterResults, 2300 onpass="In-memory counter incremented", 2301 onfail="Error incrementing in-memory" + 2302 " counter" ) 2303 2304 main.step( "Add 5 to then get a in-memory counter on each node" ) 2305 iCounters = [] 2306 threads = [] 2307 addedIValues = [] 2308 for i in range( main.numCtrls ): 2309 t = main.Thread( target=main.CLIs[i].counterTestAddAndGet, 2310 name="counterIncrement-" + str( i ), 2311 args=[ iCounterName ], 2312 kwargs={ "delta": 5, "inMemory": True } ) 2313 iCounterValue += 5 2314 addedIValues.append( iCounterValue ) 2315 threads.append( t ) 2316 t.start() 2317 2318 for t in threads: 2319 t.join() 2320 iCounters.append( t.result ) 2321 # Check that counter incremented numController times 2322 iCounterResults = True 2323 for i in addedIValues: 2324 tmpResult = i in iCounters 2325 iCounterResults = iCounterResults and tmpResult 2326 if not tmpResult: 2327 main.log.error( str( i ) + " is not in in-memory " 2328 "counter incremented results" ) 2329 utilities.assert_equals( expect=True, 2330 actual=pCounterResults, 2331 onpass="In-memory counter incremented", 2332 onfail="Error incrementing in-memory" + 2333 " counter" ) 2334 2335 main.step( "Get then add 5 to a in-memory counter on each node" ) 2336 iCounters = [] 2337 threads = [] 2338 addedIValues = [] 2339 for i in range( main.numCtrls ): 2340 t = main.Thread( target=main.CLIs[i].counterTestGetAndAdd, 2341 name="counterIncrement-" + str( i ), 2342 args=[ iCounterName ], 2343 kwargs={ "delta": 5, "inMemory": True } ) 2344 addedIValues.append( iCounterValue ) 2345 iCounterValue += 5 2346 threads.append( t ) 2347 t.start() 2348 2349 for t in threads: 2350 t.join() 2351 iCounters.append( t.result ) 2352 # Check that counter incremented numController times 2353 iCounterResults = True 2354 for i in addedIValues: 2355 tmpResult = i in iCounters 2356 iCounterResults = iCounterResults and tmpResult 2357 if not tmpResult: 2358 main.log.error( str( i ) + " is not in in-memory " 2359 "counter incremented results" ) 2360 utilities.assert_equals( expect=True, 2361 actual=iCounterResults, 2362 onpass="In-memory counter incremented", 2363 onfail="Error incrementing in-memory" + 2364 " counter" ) 2365 2366 main.step( "Counters we added have the correct values" ) 2367 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 2368 utilities.assert_equals( expect=main.TRUE, 2369 actual=incrementCheck, 2370 onpass="Added counters are correct", 2371 onfail="Added counters are incorrect" ) 2372 2373 main.step( "Check counters are consistant across nodes" ) 2374 onosCounters = [] 2375 threads = [] 2376 for i in range( main.numCtrls ): 2377 t = main.Thread( target=main.CLIs[i].counters, 2378 name="counters-" + str( i ) ) 2379 threads.append( t ) 2380 t.start() 2381 for t in threads: 2382 t.join() 2383 onosCounters.append( t.result ) 2384 tmp = [ i == onosCounters[ 0 ] for i in onosCounters ] 2385 if all( tmp ): 2386 main.log.info( "Counters are consistent across all nodes" ) 2387 consistentCounterResults = main.TRUE 2388 else: 2389 main.log.error( "Counters are not consistent across all nodes" ) 2390 consistentCounterResults = main.FALSE 2391 utilities.assert_equals( expect=main.TRUE, 2392 actual=consistentCounterResults, 2393 onpass="ONOS counters are consistent " + 2394 "across nodes", 2395 onfail="ONOS Counters are inconsistent " + 2396 "across nodes" ) 2397 2398 main.step( "Counters we added have the correct values" ) 2399 incrementCheck = main.Counters.counterCheck( iCounterName, iCounterValue ) 2400 incrementCheck = incrementCheck and \ 2401 main.Counters.counterCheck( iCounterName, iCounterValue ) 2402 utilities.assert_equals( expect=main.TRUE, 2403 actual=incrementCheck, 2404 onpass="Added counters are correct", 2405 onfail="Added counters are incorrect" ) 2406 2407 # DISTRIBUTED SETS 2408 main.step( "Distributed Set get" ) 2409 size = len( onosSet ) 2410 getResponses = [] 2411 threads = [] 2412 for i in range( main.numCtrls ): 2413 t = main.Thread( target=main.CLIs[i].setTestGet, 2414 name="setTestGet-" + str( i ), 2415 args=[ onosSetName ] ) 2416 threads.append( t ) 2417 t.start() 2418 for t in threads: 2419 t.join() 2420 getResponses.append( t.result ) 2421 2422 getResults = main.TRUE 2423 for i in range( main.numCtrls ): 2424 if isinstance( getResponses[ i ], list): 2425 current = set( getResponses[ i ] ) 2426 if len( current ) == len( getResponses[ i ] ): 2427 # no repeats 2428 if onosSet != current: 2429 main.log.error( "ONOS" + str( i + 1 ) + 2430 " has incorrect view" + 2431 " of set " + onosSetName + ":\n" + 2432 str( getResponses[ i ] ) ) 2433 main.log.debug( "Expected: " + str( onosSet ) ) 2434 main.log.debug( "Actual: " + str( current ) ) 2435 getResults = main.FALSE 2436 else: 2437 # error, set is not a set 2438 main.log.error( "ONOS" + str( i + 1 ) + 2439 " has repeat elements in" + 2440 " set " + onosSetName + ":\n" + 2441 str( getResponses[ i ] ) ) 2442 getResults = main.FALSE 2443 elif getResponses[ i ] == main.ERROR: 2444 getResults = main.FALSE 2445 utilities.assert_equals( expect=main.TRUE, 2446 actual=getResults, 2447 onpass="Set elements are correct", 2448 onfail="Set elements are incorrect" ) 2449 2450 main.step( "Distributed Set size" ) 2451 sizeResponses = [] 2452 threads = [] 2453 for i in range( main.numCtrls ): 2454 t = main.Thread( target=main.CLIs[i].setTestSize, 2455 name="setTestSize-" + str( i ), 2456 args=[ onosSetName ] ) 2457 threads.append( t ) 2458 t.start() 2459 for t in threads: 2460 t.join() 2461 sizeResponses.append( t.result ) 2462 2463 sizeResults = main.TRUE 2464 for i in range( main.numCtrls ): 2465 if size != sizeResponses[ i ]: 2466 sizeResults = main.FALSE 2467 main.log.error( "ONOS" + str( i + 1 ) + 2468 " expected a size of " + str( size ) + 2469 " for set " + onosSetName + 2470 " but got " + str( sizeResponses[ i ] ) ) 2471 utilities.assert_equals( expect=main.TRUE, 2472 actual=sizeResults, 2473 onpass="Set sizes are correct", 2474 onfail="Set sizes are incorrect" ) 2475 2476 main.step( "Distributed Set add()" ) 2477 onosSet.add( addValue ) 2478 addResponses = [] 2479 threads = [] 2480 for i in range( main.numCtrls ): 2481 t = main.Thread( target=main.CLIs[i].setTestAdd, 2482 name="setTestAdd-" + str( i ), 2483 args=[ onosSetName, addValue ] ) 2484 threads.append( t ) 2485 t.start() 2486 for t in threads: 2487 t.join() 2488 addResponses.append( t.result ) 2489 2490 # main.TRUE = successfully changed the set 2491 # main.FALSE = action resulted in no change in set 2492 # main.ERROR - Some error in executing the function 2493 addResults = main.TRUE 2494 for i in range( main.numCtrls ): 2495 if addResponses[ i ] == main.TRUE: 2496 # All is well 2497 pass 2498 elif addResponses[ i ] == main.FALSE: 2499 # Already in set, probably fine 2500 pass 2501 elif addResponses[ i ] == main.ERROR: 2502 # Error in execution 2503 addResults = main.FALSE 2504 else: 2505 # unexpected result 2506 addResults = main.FALSE 2507 if addResults != main.TRUE: 2508 main.log.error( "Error executing set add" ) 2509 2510 # Check if set is still correct 2511 size = len( onosSet ) 2512 getResponses = [] 2513 threads = [] 2514 for i in range( main.numCtrls ): 2515 t = main.Thread( target=main.CLIs[i].setTestGet, 2516 name="setTestGet-" + str( i ), 2517 args=[ onosSetName ] ) 2518 threads.append( t ) 2519 t.start() 2520 for t in threads: 2521 t.join() 2522 getResponses.append( t.result ) 2523 getResults = main.TRUE 2524 for i in range( main.numCtrls ): 2525 if isinstance( getResponses[ i ], list): 2526 current = set( getResponses[ i ] ) 2527 if len( current ) == len( getResponses[ i ] ): 2528 # no repeats 2529 if onosSet != current: 2530 main.log.error( "ONOS" + str( i + 1 ) + 2531 " has incorrect view" + 2532 " of set " + onosSetName + ":\n" + 2533 str( getResponses[ i ] ) ) 2534 main.log.debug( "Expected: " + str( onosSet ) ) 2535 main.log.debug( "Actual: " + str( current ) ) 2536 getResults = main.FALSE 2537 else: 2538 # error, set is not a set 2539 main.log.error( "ONOS" + str( i + 1 ) + 2540 " has repeat elements in" + 2541 " set " + onosSetName + ":\n" + 2542 str( getResponses[ i ] ) ) 2543 getResults = main.FALSE 2544 elif getResponses[ i ] == main.ERROR: 2545 getResults = main.FALSE 2546 sizeResponses = [] 2547 threads = [] 2548 for i in range( main.numCtrls ): 2549 t = main.Thread( target=main.CLIs[i].setTestSize, 2550 name="setTestSize-" + str( i ), 2551 args=[ onosSetName ] ) 2552 threads.append( t ) 2553 t.start() 2554 for t in threads: 2555 t.join() 2556 sizeResponses.append( t.result ) 2557 sizeResults = main.TRUE 2558 for i in range( main.numCtrls ): 2559 if size != sizeResponses[ i ]: 2560 sizeResults = main.FALSE 2561 main.log.error( "ONOS" + str( i + 1 ) + 2562 " expected a size of " + str( size ) + 2563 " for set " + onosSetName + 2564 " but got " + str( sizeResponses[ i ] ) ) 2565 addResults = addResults and getResults and sizeResults 2566 utilities.assert_equals( expect=main.TRUE, 2567 actual=addResults, 2568 onpass="Set add correct", 2569 onfail="Set add was incorrect" ) 2570 2571 main.step( "Distributed Set addAll()" ) 2572 onosSet.update( addAllValue.split() ) 2573 addResponses = [] 2574 threads = [] 2575 for i in range( main.numCtrls ): 2576 t = main.Thread( target=main.CLIs[i].setTestAdd, 2577 name="setTestAddAll-" + str( i ), 2578 args=[ onosSetName, addAllValue ] ) 2579 threads.append( t ) 2580 t.start() 2581 for t in threads: 2582 t.join() 2583 addResponses.append( t.result ) 2584 2585 # main.TRUE = successfully changed the set 2586 # main.FALSE = action resulted in no change in set 2587 # main.ERROR - Some error in executing the function 2588 addAllResults = main.TRUE 2589 for i in range( main.numCtrls ): 2590 if addResponses[ i ] == main.TRUE: 2591 # All is well 2592 pass 2593 elif addResponses[ i ] == main.FALSE: 2594 # Already in set, probably fine 2595 pass 2596 elif addResponses[ i ] == main.ERROR: 2597 # Error in execution 2598 addAllResults = main.FALSE 2599 else: 2600 # unexpected result 2601 addAllResults = main.FALSE 2602 if addAllResults != main.TRUE: 2603 main.log.error( "Error executing set addAll" ) 2604 2605 # Check if set is still correct 2606 size = len( onosSet ) 2607 getResponses = [] 2608 threads = [] 2609 for i in range( main.numCtrls ): 2610 t = main.Thread( target=main.CLIs[i].setTestGet, 2611 name="setTestGet-" + str( i ), 2612 args=[ onosSetName ] ) 2613 threads.append( t ) 2614 t.start() 2615 for t in threads: 2616 t.join() 2617 getResponses.append( t.result ) 2618 getResults = main.TRUE 2619 for i in range( main.numCtrls ): 2620 if isinstance( getResponses[ i ], list): 2621 current = set( getResponses[ i ] ) 2622 if len( current ) == len( getResponses[ i ] ): 2623 # no repeats 2624 if onosSet != current: 2625 main.log.error( "ONOS" + str( i + 1 ) + 2626 " has incorrect view" + 2627 " of set " + onosSetName + ":\n" + 2628 str( getResponses[ i ] ) ) 2629 main.log.debug( "Expected: " + str( onosSet ) ) 2630 main.log.debug( "Actual: " + str( current ) ) 2631 getResults = main.FALSE 2632 else: 2633 # error, set is not a set 2634 main.log.error( "ONOS" + str( i + 1 ) + 2635 " has repeat elements in" + 2636 " set " + onosSetName + ":\n" + 2637 str( getResponses[ i ] ) ) 2638 getResults = main.FALSE 2639 elif getResponses[ i ] == main.ERROR: 2640 getResults = main.FALSE 2641 sizeResponses = [] 2642 threads = [] 2643 for i in range( main.numCtrls ): 2644 t = main.Thread( target=main.CLIs[i].setTestSize, 2645 name="setTestSize-" + str( i ), 2646 args=[ onosSetName ] ) 2647 threads.append( t ) 2648 t.start() 2649 for t in threads: 2650 t.join() 2651 sizeResponses.append( t.result ) 2652 sizeResults = main.TRUE 2653 for i in range( main.numCtrls ): 2654 if size != sizeResponses[ i ]: 2655 sizeResults = main.FALSE 2656 main.log.error( "ONOS" + str( i + 1 ) + 2657 " expected a size of " + str( size ) + 2658 " for set " + onosSetName + 2659 " but got " + str( sizeResponses[ i ] ) ) 2660 addAllResults = addAllResults and getResults and sizeResults 2661 utilities.assert_equals( expect=main.TRUE, 2662 actual=addAllResults, 2663 onpass="Set addAll correct", 2664 onfail="Set addAll was incorrect" ) 2665 2666 main.step( "Distributed Set contains()" ) 2667 containsResponses = [] 2668 threads = [] 2669 for i in range( main.numCtrls ): 2670 t = main.Thread( target=main.CLIs[i].setTestGet, 2671 name="setContains-" + str( i ), 2672 args=[ onosSetName ], 2673 kwargs={ "values": addValue } ) 2674 threads.append( t ) 2675 t.start() 2676 for t in threads: 2677 t.join() 2678 # NOTE: This is the tuple 2679 containsResponses.append( t.result ) 2680 2681 containsResults = main.TRUE 2682 for i in range( main.numCtrls ): 2683 if containsResponses[ i ] == main.ERROR: 2684 containsResults = main.FALSE 2685 else: 2686 containsResults = containsResults and\ 2687 containsResponses[ i ][ 1 ] 2688 utilities.assert_equals( expect=main.TRUE, 2689 actual=containsResults, 2690 onpass="Set contains is functional", 2691 onfail="Set contains failed" ) 2692 2693 main.step( "Distributed Set containsAll()" ) 2694 containsAllResponses = [] 2695 threads = [] 2696 for i in range( main.numCtrls ): 2697 t = main.Thread( target=main.CLIs[i].setTestGet, 2698 name="setContainsAll-" + str( i ), 2699 args=[ onosSetName ], 2700 kwargs={ "values": addAllValue } ) 2701 threads.append( t ) 2702 t.start() 2703 for t in threads: 2704 t.join() 2705 # NOTE: This is the tuple 2706 containsAllResponses.append( t.result ) 2707 2708 containsAllResults = main.TRUE 2709 for i in range( main.numCtrls ): 2710 if containsResponses[ i ] == main.ERROR: 2711 containsResults = main.FALSE 2712 else: 2713 containsResults = containsResults and\ 2714 containsResponses[ i ][ 1 ] 2715 utilities.assert_equals( expect=main.TRUE, 2716 actual=containsAllResults, 2717 onpass="Set containsAll is functional", 2718 onfail="Set containsAll failed" ) 2719 2720 main.step( "Distributed Set remove()" ) 2721 onosSet.remove( addValue ) 2722 removeResponses = [] 2723 threads = [] 2724 for i in range( main.numCtrls ): 2725 t = main.Thread( target=main.CLIs[i].setTestRemove, 2726 name="setTestRemove-" + str( i ), 2727 args=[ onosSetName, addValue ] ) 2728 threads.append( t ) 2729 t.start() 2730 for t in threads: 2731 t.join() 2732 removeResponses.append( t.result ) 2733 2734 # main.TRUE = successfully changed the set 2735 # main.FALSE = action resulted in no change in set 2736 # main.ERROR - Some error in executing the function 2737 removeResults = main.TRUE 2738 for i in range( main.numCtrls ): 2739 if removeResponses[ i ] == main.TRUE: 2740 # All is well 2741 pass 2742 elif removeResponses[ i ] == main.FALSE: 2743 # not in set, probably fine 2744 pass 2745 elif removeResponses[ i ] == main.ERROR: 2746 # Error in execution 2747 removeResults = main.FALSE 2748 else: 2749 # unexpected result 2750 removeResults = main.FALSE 2751 if removeResults != main.TRUE: 2752 main.log.error( "Error executing set remove" ) 2753 2754 # Check if set is still correct 2755 size = len( onosSet ) 2756 getResponses = [] 2757 threads = [] 2758 for i in range( main.numCtrls ): 2759 t = main.Thread( target=main.CLIs[i].setTestGet, 2760 name="setTestGet-" + str( i ), 2761 args=[ onosSetName ] ) 2762 threads.append( t ) 2763 t.start() 2764 for t in threads: 2765 t.join() 2766 getResponses.append( t.result ) 2767 getResults = main.TRUE 2768 for i in range( main.numCtrls ): 2769 if isinstance( getResponses[ i ], list): 2770 current = set( getResponses[ i ] ) 2771 if len( current ) == len( getResponses[ i ] ): 2772 # no repeats 2773 if onosSet != current: 2774 main.log.error( "ONOS" + str( i + 1 ) + 2775 " has incorrect view" + 2776 " of set " + onosSetName + ":\n" + 2777 str( getResponses[ i ] ) ) 2778 main.log.debug( "Expected: " + str( onosSet ) ) 2779 main.log.debug( "Actual: " + str( current ) ) 2780 getResults = main.FALSE 2781 else: 2782 # error, set is not a set 2783 main.log.error( "ONOS" + str( i + 1 ) + 2784 " has repeat elements in" + 2785 " set " + onosSetName + ":\n" + 2786 str( getResponses[ i ] ) ) 2787 getResults = main.FALSE 2788 elif getResponses[ i ] == main.ERROR: 2789 getResults = main.FALSE 2790 sizeResponses = [] 2791 threads = [] 2792 for i in range( main.numCtrls ): 2793 t = main.Thread( target=main.CLIs[i].setTestSize, 2794 name="setTestSize-" + str( i ), 2795 args=[ onosSetName ] ) 2796 threads.append( t ) 2797 t.start() 2798 for t in threads: 2799 t.join() 2800 sizeResponses.append( t.result ) 2801 sizeResults = main.TRUE 2802 for i in range( main.numCtrls ): 2803 if size != sizeResponses[ i ]: 2804 sizeResults = main.FALSE 2805 main.log.error( "ONOS" + str( i + 1 ) + 2806 " expected a size of " + str( size ) + 2807 " for set " + onosSetName + 2808 " but got " + str( sizeResponses[ i ] ) ) 2809 removeResults = removeResults and getResults and sizeResults 2810 utilities.assert_equals( expect=main.TRUE, 2811 actual=removeResults, 2812 onpass="Set remove correct", 2813 onfail="Set remove was incorrect" ) 2814 2815 main.step( "Distributed Set removeAll()" ) 2816 onosSet.difference_update( addAllValue.split() ) 2817 removeAllResponses = [] 2818 threads = [] 2819 try: 2820 for i in range( main.numCtrls ): 2821 t = main.Thread( target=main.CLIs[i].setTestRemove, 2822 name="setTestRemoveAll-" + str( i ), 2823 args=[ onosSetName, addAllValue ] ) 2824 threads.append( t ) 2825 t.start() 2826 for t in threads: 2827 t.join() 2828 removeAllResponses.append( t.result ) 2829 except Exception, e: 2830 main.log.exception(e) 2831 2832 # main.TRUE = successfully changed the set 2833 # main.FALSE = action resulted in no change in set 2834 # main.ERROR - Some error in executing the function 2835 removeAllResults = main.TRUE 2836 for i in range( main.numCtrls ): 2837 if removeAllResponses[ i ] == main.TRUE: 2838 # All is well 2839 pass 2840 elif removeAllResponses[ i ] == main.FALSE: 2841 # not in set, probably fine 2842 pass 2843 elif removeAllResponses[ i ] == main.ERROR: 2844 # Error in execution 2845 removeAllResults = main.FALSE 2846 else: 2847 # unexpected result 2848 removeAllResults = main.FALSE 2849 if removeAllResults != main.TRUE: 2850 main.log.error( "Error executing set removeAll" ) 2851 2852 # Check if set is still correct 2853 size = len( onosSet ) 2854 getResponses = [] 2855 threads = [] 2856 for i in range( main.numCtrls ): 2857 t = main.Thread( target=main.CLIs[i].setTestGet, 2858 name="setTestGet-" + str( i ), 2859 args=[ onosSetName ] ) 2860 threads.append( t ) 2861 t.start() 2862 for t in threads: 2863 t.join() 2864 getResponses.append( t.result ) 2865 getResults = main.TRUE 2866 for i in range( main.numCtrls ): 2867 if isinstance( getResponses[ i ], list): 2868 current = set( getResponses[ i ] ) 2869 if len( current ) == len( getResponses[ i ] ): 2870 # no repeats 2871 if onosSet != current: 2872 main.log.error( "ONOS" + str( i + 1 ) + 2873 " has incorrect view" + 2874 " of set " + onosSetName + ":\n" + 2875 str( getResponses[ i ] ) ) 2876 main.log.debug( "Expected: " + str( onosSet ) ) 2877 main.log.debug( "Actual: " + str( current ) ) 2878 getResults = main.FALSE 2879 else: 2880 # error, set is not a set 2881 main.log.error( "ONOS" + str( i + 1 ) + 2882 " has repeat elements in" + 2883 " set " + onosSetName + ":\n" + 2884 str( getResponses[ i ] ) ) 2885 getResults = main.FALSE 2886 elif getResponses[ i ] == main.ERROR: 2887 getResults = main.FALSE 2888 sizeResponses = [] 2889 threads = [] 2890 for i in range( main.numCtrls ): 2891 t = main.Thread( target=main.CLIs[i].setTestSize, 2892 name="setTestSize-" + str( i ), 2893 args=[ onosSetName ] ) 2894 threads.append( t ) 2895 t.start() 2896 for t in threads: 2897 t.join() 2898 sizeResponses.append( t.result ) 2899 sizeResults = main.TRUE 2900 for i in range( main.numCtrls ): 2901 if size != sizeResponses[ i ]: 2902 sizeResults = main.FALSE 2903 main.log.error( "ONOS" + str( i + 1 ) + 2904 " expected a size of " + str( size ) + 2905 " for set " + onosSetName + 2906 " but got " + str( sizeResponses[ i ] ) ) 2907 removeAllResults = removeAllResults and getResults and sizeResults 2908 utilities.assert_equals( expect=main.TRUE, 2909 actual=removeAllResults, 2910 onpass="Set removeAll correct", 2911 onfail="Set removeAll was incorrect" ) 2912 2913 main.step( "Distributed Set addAll()" ) 2914 onosSet.update( addAllValue.split() ) 2915 addResponses = [] 2916 threads = [] 2917 for i in range( main.numCtrls ): 2918 t = main.Thread( target=main.CLIs[i].setTestAdd, 2919 name="setTestAddAll-" + str( i ), 2920 args=[ onosSetName, addAllValue ] ) 2921 threads.append( t ) 2922 t.start() 2923 for t in threads: 2924 t.join() 2925 addResponses.append( t.result ) 2926 2927 # main.TRUE = successfully changed the set 2928 # main.FALSE = action resulted in no change in set 2929 # main.ERROR - Some error in executing the function 2930 addAllResults = main.TRUE 2931 for i in range( main.numCtrls ): 2932 if addResponses[ i ] == main.TRUE: 2933 # All is well 2934 pass 2935 elif addResponses[ i ] == main.FALSE: 2936 # Already in set, probably fine 2937 pass 2938 elif addResponses[ i ] == main.ERROR: 2939 # Error in execution 2940 addAllResults = main.FALSE 2941 else: 2942 # unexpected result 2943 addAllResults = main.FALSE 2944 if addAllResults != main.TRUE: 2945 main.log.error( "Error executing set addAll" ) 2946 2947 # Check if set is still correct 2948 size = len( onosSet ) 2949 getResponses = [] 2950 threads = [] 2951 for i in range( main.numCtrls ): 2952 t = main.Thread( target=main.CLIs[i].setTestGet, 2953 name="setTestGet-" + str( i ), 2954 args=[ onosSetName ] ) 2955 threads.append( t ) 2956 t.start() 2957 for t in threads: 2958 t.join() 2959 getResponses.append( t.result ) 2960 getResults = main.TRUE 2961 for i in range( main.numCtrls ): 2962 if isinstance( getResponses[ i ], list): 2963 current = set( getResponses[ i ] ) 2964 if len( current ) == len( getResponses[ i ] ): 2965 # no repeats 2966 if onosSet != current: 2967 main.log.error( "ONOS" + str( i + 1 ) + 2968 " has incorrect view" + 2969 " of set " + onosSetName + ":\n" + 2970 str( getResponses[ i ] ) ) 2971 main.log.debug( "Expected: " + str( onosSet ) ) 2972 main.log.debug( "Actual: " + str( current ) ) 2973 getResults = main.FALSE 2974 else: 2975 # error, set is not a set 2976 main.log.error( "ONOS" + str( i + 1 ) + 2977 " has repeat elements in" + 2978 " set " + onosSetName + ":\n" + 2979 str( getResponses[ i ] ) ) 2980 getResults = main.FALSE 2981 elif getResponses[ i ] == main.ERROR: 2982 getResults = main.FALSE 2983 sizeResponses = [] 2984 threads = [] 2985 for i in range( main.numCtrls ): 2986 t = main.Thread( target=main.CLIs[i].setTestSize, 2987 name="setTestSize-" + str( i ), 2988 args=[ onosSetName ] ) 2989 threads.append( t ) 2990 t.start() 2991 for t in threads: 2992 t.join() 2993 sizeResponses.append( t.result ) 2994 sizeResults = main.TRUE 2995 for i in range( main.numCtrls ): 2996 if size != sizeResponses[ i ]: 2997 sizeResults = main.FALSE 2998 main.log.error( "ONOS" + str( i + 1 ) + 2999 " expected a size of " + str( size ) + 3000 " for set " + onosSetName + 3001 " but got " + str( sizeResponses[ i ] ) ) 3002 addAllResults = addAllResults and getResults and sizeResults 3003 utilities.assert_equals( expect=main.TRUE, 3004 actual=addAllResults, 3005 onpass="Set addAll correct", 3006 onfail="Set addAll was incorrect" ) 3007 3008 main.step( "Distributed Set clear()" ) 3009 onosSet.clear() 3010 clearResponses = [] 3011 threads = [] 3012 for i in range( main.numCtrls ): 3013 t = main.Thread( target=main.CLIs[i].setTestRemove, 3014 name="setTestClear-" + str( i ), 3015 args=[ onosSetName, " "], # Values doesn't matter 3016 kwargs={ "clear": True } ) 3017 threads.append( t ) 3018 t.start() 3019 for t in threads: 3020 t.join() 3021 clearResponses.append( t.result ) 3022 3023 # main.TRUE = successfully changed the set 3024 # main.FALSE = action resulted in no change in set 3025 # main.ERROR - Some error in executing the function 3026 clearResults = main.TRUE 3027 for i in range( main.numCtrls ): 3028 if clearResponses[ i ] == main.TRUE: 3029 # All is well 3030 pass 3031 elif clearResponses[ i ] == main.FALSE: 3032 # Nothing set, probably fine 3033 pass 3034 elif clearResponses[ i ] == main.ERROR: 3035 # Error in execution 3036 clearResults = main.FALSE 3037 else: 3038 # unexpected result 3039 clearResults = main.FALSE 3040 if clearResults != main.TRUE: 3041 main.log.error( "Error executing set clear" ) 3042 3043 # Check if set is still correct 3044 size = len( onosSet ) 3045 getResponses = [] 3046 threads = [] 3047 for i in range( main.numCtrls ): 3048 t = main.Thread( target=main.CLIs[i].setTestGet, 3049 name="setTestGet-" + str( i ), 3050 args=[ onosSetName ] ) 3051 threads.append( t ) 3052 t.start() 3053 for t in threads: 3054 t.join() 3055 getResponses.append( t.result ) 3056 getResults = main.TRUE 3057 for i in range( main.numCtrls ): 3058 if isinstance( getResponses[ i ], list): 3059 current = set( getResponses[ i ] ) 3060 if len( current ) == len( getResponses[ i ] ): 3061 # no repeats 3062 if onosSet != current: 3063 main.log.error( "ONOS" + str( i + 1 ) + 3064 " has incorrect view" + 3065 " of set " + onosSetName + ":\n" + 3066 str( getResponses[ i ] ) ) 3067 main.log.debug( "Expected: " + str( onosSet ) ) 3068 main.log.debug( "Actual: " + str( current ) ) 3069 getResults = main.FALSE 3070 else: 3071 # error, set is not a set 3072 main.log.error( "ONOS" + str( i + 1 ) + 3073 " has repeat elements in" + 3074 " set " + onosSetName + ":\n" + 3075 str( getResponses[ i ] ) ) 3076 getResults = main.FALSE 3077 elif getResponses[ i ] == main.ERROR: 3078 getResults = main.FALSE 3079 sizeResponses = [] 3080 threads = [] 3081 for i in range( main.numCtrls ): 3082 t = main.Thread( target=main.CLIs[i].setTestSize, 3083 name="setTestSize-" + str( i ), 3084 args=[ onosSetName ] ) 3085 threads.append( t ) 3086 t.start() 3087 for t in threads: 3088 t.join() 3089 sizeResponses.append( t.result ) 3090 sizeResults = main.TRUE 3091 for i in range( main.numCtrls ): 3092 if size != sizeResponses[ i ]: 3093 sizeResults = main.FALSE 3094 main.log.error( "ONOS" + str( i + 1 ) + 3095 " expected a size of " + str( size ) + 3096 " for set " + onosSetName + 3097 " but got " + str( sizeResponses[ i ] ) ) 3098 clearResults = clearResults and getResults and sizeResults 3099 utilities.assert_equals( expect=main.TRUE, 3100 actual=clearResults, 3101 onpass="Set clear correct", 3102 onfail="Set clear was incorrect" ) 3103 3104 main.step( "Distributed Set addAll()" ) 3105 onosSet.update( addAllValue.split() ) 3106 addResponses = [] 3107 threads = [] 3108 for i in range( main.numCtrls ): 3109 t = main.Thread( target=main.CLIs[i].setTestAdd, 3110 name="setTestAddAll-" + str( i ), 3111 args=[ onosSetName, addAllValue ] ) 3112 threads.append( t ) 3113 t.start() 3114 for t in threads: 3115 t.join() 3116 addResponses.append( t.result ) 3117 3118 # main.TRUE = successfully changed the set 3119 # main.FALSE = action resulted in no change in set 3120 # main.ERROR - Some error in executing the function 3121 addAllResults = main.TRUE 3122 for i in range( main.numCtrls ): 3123 if addResponses[ i ] == main.TRUE: 3124 # All is well 3125 pass 3126 elif addResponses[ i ] == main.FALSE: 3127 # Already in set, probably fine 3128 pass 3129 elif addResponses[ i ] == main.ERROR: 3130 # Error in execution 3131 addAllResults = main.FALSE 3132 else: 3133 # unexpected result 3134 addAllResults = main.FALSE 3135 if addAllResults != main.TRUE: 3136 main.log.error( "Error executing set addAll" ) 3137 3138 # Check if set is still correct 3139 size = len( onosSet ) 3140 getResponses = [] 3141 threads = [] 3142 for i in range( main.numCtrls ): 3143 t = main.Thread( target=main.CLIs[i].setTestGet, 3144 name="setTestGet-" + str( i ), 3145 args=[ onosSetName ] ) 3146 threads.append( t ) 3147 t.start() 3148 for t in threads: 3149 t.join() 3150 getResponses.append( t.result ) 3151 getResults = main.TRUE 3152 for i in range( main.numCtrls ): 3153 if isinstance( getResponses[ i ], list): 3154 current = set( getResponses[ i ] ) 3155 if len( current ) == len( getResponses[ i ] ): 3156 # no repeats 3157 if onosSet != current: 3158 main.log.error( "ONOS" + str( i + 1 ) + 3159 " has incorrect view" + 3160 " of set " + onosSetName + ":\n" + 3161 str( getResponses[ i ] ) ) 3162 main.log.debug( "Expected: " + str( onosSet ) ) 3163 main.log.debug( "Actual: " + str( current ) ) 3164 getResults = main.FALSE 3165 else: 3166 # error, set is not a set 3167 main.log.error( "ONOS" + str( i + 1 ) + 3168 " has repeat elements in" + 3169 " set " + onosSetName + ":\n" + 3170 str( getResponses[ i ] ) ) 3171 getResults = main.FALSE 3172 elif getResponses[ i ] == main.ERROR: 3173 getResults = main.FALSE 3174 sizeResponses = [] 3175 threads = [] 3176 for i in range( main.numCtrls ): 3177 t = main.Thread( target=main.CLIs[i].setTestSize, 3178 name="setTestSize-" + str( i ), 3179 args=[ onosSetName ] ) 3180 threads.append( t ) 3181 t.start() 3182 for t in threads: 3183 t.join() 3184 sizeResponses.append( t.result ) 3185 sizeResults = main.TRUE 3186 for i in range( main.numCtrls ): 3187 if size != sizeResponses[ i ]: 3188 sizeResults = main.FALSE 3189 main.log.error( "ONOS" + str( i + 1 ) + 3190 " expected a size of " + str( size ) + 3191 " for set " + onosSetName + 3192 " but got " + str( sizeResponses[ i ] ) ) 3193 addAllResults = addAllResults and getResults and sizeResults 3194 utilities.assert_equals( expect=main.TRUE, 3195 actual=addAllResults, 3196 onpass="Set addAll correct", 3197 onfail="Set addAll was incorrect" ) 3198 3199 main.step( "Distributed Set retain()" ) 3200 onosSet.intersection_update( retainValue.split() ) 3201 retainResponses = [] 3202 threads = [] 3203 for i in range( main.numCtrls ): 3204 t = main.Thread( target=main.CLIs[i].setTestRemove, 3205 name="setTestRetain-" + str( i ), 3206 args=[ onosSetName, retainValue ], 3207 kwargs={ "retain": True } ) 3208 threads.append( t ) 3209 t.start() 3210 for t in threads: 3211 t.join() 3212 retainResponses.append( t.result ) 3213 3214 # main.TRUE = successfully changed the set 3215 # main.FALSE = action resulted in no change in set 3216 # main.ERROR - Some error in executing the function 3217 retainResults = main.TRUE 3218 for i in range( main.numCtrls ): 3219 if retainResponses[ i ] == main.TRUE: 3220 # All is well 3221 pass 3222 elif retainResponses[ i ] == main.FALSE: 3223 # Already in set, probably fine 3224 pass 3225 elif retainResponses[ i ] == main.ERROR: 3226 # Error in execution 3227 retainResults = main.FALSE 3228 else: 3229 # unexpected result 3230 retainResults = main.FALSE 3231 if retainResults != main.TRUE: 3232 main.log.error( "Error executing set retain" ) 3233 3234 # Check if set is still correct 3235 size = len( onosSet ) 3236 getResponses = [] 3237 threads = [] 3238 for i in range( main.numCtrls ): 3239 t = main.Thread( target=main.CLIs[i].setTestGet, 3240 name="setTestGet-" + str( i ), 3241 args=[ onosSetName ] ) 3242 threads.append( t ) 3243 t.start() 3244 for t in threads: 3245 t.join() 3246 getResponses.append( t.result ) 3247 getResults = main.TRUE 3248 for i in range( main.numCtrls ): 3249 if isinstance( getResponses[ i ], list): 3250 current = set( getResponses[ i ] ) 3251 if len( current ) == len( getResponses[ i ] ): 3252 # no repeats 3253 if onosSet != current: 3254 main.log.error( "ONOS" + str( i + 1 ) + 3255 " has incorrect view" + 3256 " of set " + onosSetName + ":\n" + 3257 str( getResponses[ i ] ) ) 3258 main.log.debug( "Expected: " + str( onosSet ) ) 3259 main.log.debug( "Actual: " + str( current ) ) 3260 getResults = main.FALSE 3261 else: 3262 # error, set is not a set 3263 main.log.error( "ONOS" + str( i + 1 ) + 3264 " has repeat elements in" + 3265 " set " + onosSetName + ":\n" + 3266 str( getResponses[ i ] ) ) 3267 getResults = main.FALSE 3268 elif getResponses[ i ] == main.ERROR: 3269 getResults = main.FALSE 3270 sizeResponses = [] 3271 threads = [] 3272 for i in range( main.numCtrls ): 3273 t = main.Thread( target=main.CLIs[i].setTestSize, 3274 name="setTestSize-" + str( i ), 3275 args=[ onosSetName ] ) 3276 threads.append( t ) 3277 t.start() 3278 for t in threads: 3279 t.join() 3280 sizeResponses.append( t.result ) 3281 sizeResults = main.TRUE 3282 for i in range( main.numCtrls ): 3283 if size != sizeResponses[ i ]: 3284 sizeResults = main.FALSE 3285 main.log.error( "ONOS" + str( i + 1 ) + 3286 " expected a size of " + 3287 str( size ) + " for set " + onosSetName + 3288 " but got " + str( sizeResponses[ i ] ) ) 3289 retainResults = retainResults and getResults and sizeResults 3290 utilities.assert_equals( expect=main.TRUE, 3291 actual=retainResults, 3292 onpass="Set retain correct", 3293 onfail="Set retain was incorrect" )
3294