| |
| # Testing network scalability, this test suite scales up a network topology |
| # using mininet and verifies ONOS stability |
| |
| class SCPFscaleTopo: |
| |
| def __init__( self ): |
| self.default = '' |
| |
| def CASE1( self, main ): |
| import os |
| import imp |
| |
| """ |
| - Construct tests variables |
| - GIT ( optional ) |
| - Checkout ONOS master branch |
| - Pull latest ONOS code |
| - Building ONOS ( optional ) |
| - Install ONOS package |
| - Build ONOS package |
| """ |
| main.case( "Constructing test variables" ) |
| main.step( "Constructing test variables" ) |
| stepResult = main.FALSE |
| # The variable to decide if the data should be written into data base. |
| # 1 means Yes and -1 means No. |
| main.writeData = 1 |
| main.searchTerm = main.params[ 'SearchTerm' ] |
| main.testOnDirectory = os.path.dirname( os.getcwd ( ) ) |
| main.apps = main.params[ 'ENV' ][ 'cellApps' ] |
| gitBranch = main.params[ 'GIT' ][ 'branch' ] |
| main.dependencyPath = main.testOnDirectory + \ |
| main.params[ 'DEPENDENCY' ][ 'path' ] |
| main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ] |
| main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ] |
| main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] ) |
| main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," ) |
| main.topoScaleSize = len( main.topoScale ) |
| wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ] |
| wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ] |
| wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ] |
| main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] ) |
| main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] ) |
| main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] ) |
| main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] ) |
| main.nodeSleep = int( main.params[ 'SLEEP' ][ 'nodeSleep' ] ) |
| main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] ) |
| main.MNSleep = int( main.params[ 'SLEEP' ][ 'MNsleep' ] ) |
| main.pingTimeout = float( main.params[ 'TIMEOUT' ][ 'pingall' ] ) |
| main.hostDiscover = main.params[ 'TOPOLOGY' ][ 'host' ] |
| main.hostDiscoverSleep = float( main.params['SLEEP']['host'] ) |
| if main.hostDiscover == 'True': |
| main.hostDiscover = True |
| else: |
| main.hostDiscover = False |
| gitPull = main.params[ 'GIT' ][ 'pull' ] |
| main.homeDir = os.path.expanduser('~') |
| main.cellData = {} # for creating cell file |
| main.hostsData = {} |
| main.CLIs = [] |
| main.ONOSip = [] |
| main.activeNodes = [] |
| main.ONOSip = main.ONOSbench.getOnosIps() |
| |
| for i in range(main.numCtrls): |
| main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) ) |
| |
| main.allinfo = {} # The dictionary to record all the data from karaf.log |
| for i in range( 2 ): |
| main.allinfo[ i ]={} |
| for w in range ( 3 ): |
| # Totaltime: the time from the new switchConnection to its end |
| # swConnection: the time from the first new switchConnection to the last new switchConnection |
| # disconnectRate: the rate that shows how many switch disconnect after connection |
| main.allinfo[ i ][ 'info' + str( w ) ]= { 'totalTime': 0, 'swConnection': 0,'lastSwToTopology': 0, 'disconnectRate': 0 } |
| |
| main.dbFilePath = main.params[ 'DATABASE' ][ 'dbPath' ] |
| main.log.info( "Create Database file " + main.dbFilePath ) |
| resultDB = open(main.dbFilePath, 'w+' ) |
| resultDB.close() |
| |
| main.startUp = imp.load_source( wrapperFile1, |
| main.dependencyPath + |
| wrapperFile1 + |
| ".py" ) |
| main.scaleTopoFunction = imp.load_source( wrapperFile2, |
| main.dependencyPath + |
| wrapperFile2 + |
| ".py" ) |
| main.topo = imp.load_source( wrapperFile3, |
| main.dependencyPath + |
| wrapperFile3 + |
| ".py" ) |
| main.ONOSbench.scp( main.Mininet1, |
| main.dependencyPath + |
| main.multiovs, |
| main.Mininet1.home, |
| direction="to" ) |
| |
| if main.CLIs: |
| stepResult = main.TRUE |
| else: |
| main.log.error( "Did not properly created list of " + |
| "ONOS CLI handle" ) |
| stepResult = main.FALSE |
| |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Successfully construct " + |
| "test variables ", |
| onfail="Failed to construct test variables" ) |
| |
| if gitPull == 'True': |
| main.step( "Building ONOS in " + gitBranch + " branch" ) |
| onosBuildResult = main.startUp.onosBuild( main, gitBranch ) |
| stepResult = onosBuildResult |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Successfully compiled " + |
| "latest ONOS", |
| onfail="Failed to compile " + |
| "latest ONOS" ) |
| else: |
| main.log.warn( "Did not pull new code so skipping mvn " + |
| "clean install" ) |
| |
| |
| def CASE2( self, main): |
| """ |
| - Set up cell |
| - Create cell file |
| - Set cell file |
| - Verify cell file |
| - Kill ONOS process |
| - Uninstall ONOS cluster |
| - Verify ONOS start up |
| - Install ONOS cluster |
| - Connect to cli |
| """ |
| import time |
| main.log.info( "Checking if mininet is already running" ) |
| if len( main.topoScale ) < main.topoScaleSize: |
| main.log.info( "Mininet is already running. Stopping mininet." ) |
| main.Mininet1.stopNet() |
| time.sleep(main.MNSleep) |
| else: |
| main.log.info( "Mininet was not running" ) |
| |
| main.case( "Starting up " + str( main.numCtrls ) + |
| " node(s) ONOS cluster" ) |
| main.caseExplanation = "Set up ONOS with " + str( main.numCtrls ) +\ |
| " node(s) ONOS cluster" |
| |
| |
| |
| #kill off all onos processes |
| main.log.info( "Safety check, killing all ONOS processes" + |
| " before initiating environment setup" ) |
| |
| for i in range( main.numCtrls ): |
| main.ONOSbench.onosDie( main.ONOSip[ i ] ) |
| |
| tempOnosIp = [] |
| for i in range( main.numCtrls ): |
| tempOnosIp.append( main.ONOSip[i] ) |
| |
| main.ONOSbench.createCellFile( main.ONOSbench.ip_address, |
| "temp", main.Mininet1.ip_address, |
| main.apps, tempOnosIp ) |
| |
| main.step( "Apply cell to environment" ) |
| cellResult = main.ONOSbench.setCell( "temp" ) |
| verifyResult = main.ONOSbench.verifyCell() |
| stepResult = cellResult and verifyResult |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Successfully applied cell to " + \ |
| "environment", |
| onfail="Failed to apply cell to environment " ) |
| |
| main.step( "Creating ONOS package" ) |
| packageResult = main.ONOSbench.buckBuild() |
| stepResult = packageResult |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Successfully created ONOS package", |
| onfail="Failed to create ONOS package" ) |
| |
| time.sleep( main.startUpSleep ) |
| main.step( "Uninstalling ONOS package" ) |
| onosUninstallResult = main.TRUE |
| for ip in main.ONOSip: |
| onosUninstallResult = onosUninstallResult and \ |
| main.ONOSbench.onosUninstall( nodeIp=ip ) |
| stepResult = onosUninstallResult |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Successfully uninstalled ONOS package", |
| onfail="Failed to uninstall ONOS package" ) |
| |
| time.sleep( main.startUpSleep ) |
| main.step( "Installing ONOS package" ) |
| onosInstallResult = main.TRUE |
| for i in range( main.numCtrls ): |
| onosInstallResult = onosInstallResult and \ |
| main.ONOSbench.onosInstall( node=main.ONOSip[ i ] ) |
| stepResult = onosInstallResult |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Successfully installed ONOS package", |
| onfail="Failed to install ONOS package" ) |
| |
| time.sleep( main.startUpSleep ) |
| main.step( "Starting ONOS service" ) |
| stopResult = main.TRUE |
| startResult = main.TRUE |
| onosIsUp = main.TRUE |
| |
| for i in range( main.numCtrls ): |
| onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] ) |
| if onosIsUp == main.TRUE: |
| main.log.report( "ONOS instance is up and ready" ) |
| else: |
| main.log.report( "ONOS instance may not be up, stop and " + |
| "start ONOS again " ) |
| |
| for i in range( main.numCtrls ): |
| stopResult = stopResult and \ |
| main.ONOSbench.onosStop( main.ONOSip[ i ] ) |
| for i in range( main.numCtrls ): |
| startResult = startResult and \ |
| main.ONOSbench.onosStart( main.ONOSip[ i ] ) |
| stepResult = onosIsUp and stopResult and startResult |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="ONOS service is ready", |
| onfail="ONOS service did not start properly" ) |
| |
| main.step( "Set up ONOS secure SSH" ) |
| secureSshResult = main.TRUE |
| for i in range( int( main.numCtrls ) ): |
| secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=main.ONOSip[i] ) |
| utilities.assert_equals( expect=main.TRUE, actual=secureSshResult, |
| onpass="Test step PASS", |
| onfail="Test step FAIL" ) |
| |
| main.step( "Start ONOS cli" ) |
| cliResult = main.TRUE |
| main.activeNodes = [] |
| for i in range( main.numCtrls ): |
| cliResult = cliResult and \ |
| main.CLIs[ i ].startOnosCli( main.ONOSip[ i ] ) |
| main.activeNodes.append( i ) |
| stepResult = cliResult |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Successfully start ONOS cli", |
| onfail="Failed to start ONOS cli" ) |
| time.sleep( main.startUpSleep ) |
| |
| def CASE10( self, main ): |
| """ |
| Starting up torus topology |
| """ |
| |
| main.case( "Starting up Mininet and verifying topology" ) |
| main.caseExplanation = "Starting Mininet with a scalling topology and " +\ |
| "comparing topology elements between Mininet and ONOS" |
| if main.topoScale: |
| main.currScale = main.topoScale.pop(0) |
| else: main.log.error( "topology scale is empty" ) |
| main.step( "Starting up TORUS %sx%s topology" % (main.currScale, main.currScale) ) |
| |
| main.log.info( "Constructing Mininet command" ) |
| mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs + \ |
| " --switch ovsm --topo " + main.topoName + "," + main.currScale + "," + main.currScale |
| for i in range( main.numCtrls ): |
| mnCmd += " --controller remote,ip=" + main.ONOSip[ i ] |
| stepResult = main.Mininet1.startNet( mnCmd=mnCmd ) |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass=main.topoName + |
| " topology started successfully", |
| onfail=main.topoName + |
| " topology failed to start" ) |
| |
| time.sleep( main.MNSleep ) |
| main.CLIs[ 0 ].activateApp( "org.onosproject.openflow" ) |
| time.sleep( main.MNSleep ) |
| |
| def CASE11( self, main ): |
| """ |
| Compare topo, and sending Arping package |
| if the topology is same, then Pass. |
| """ |
| import json |
| import time |
| # First capture |
| for i in range( 3 ): |
| # Calculate total time |
| main.allinfo[ 0 ][ 'info' + str( i )][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'end' ], 'last', index=i, funcMode='TD' ) |
| # Calculate switch connection time |
| main.allinfo[ 0 ][ 'info' + str( i )][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'start' ], 'last', index=i, funcMode='TD' ) |
| # Calculate the time from last switch connection to the end |
| main.allinfo[ 0 ][ 'info' + str( i )][ 'lastSwToTopology' ] = main.allinfo[ 0 ][ 'info' + str( i )][ 'totalTime' ] - main.allinfo[ 0 ][ 'info' + str( i )][ 'swConnection' ] |
| # Calculate the disconnecti rate |
| main.allinfo[ 0 ][ 'info' + str( i )][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'Disconnect' ], 'num', main.searchTerm[ 'start' ], 'num', index=i, funcMode='DR' ) |
| main.log.debug( "The data is " + str( main.allinfo[ 0 ] ) ) |
| |
| main.case( "Verifying topology: TORUS %sx%s" % ( main.currScale, main.currScale ) ) |
| main.caseExplanation = "Pinging all hosts and comparing topology " +\ |
| "elements between Mininet and ONOS" |
| |
| main.log.info( "Gathering topology information") |
| time.sleep( main.MNSleep ) |
| stepResult = main.TRUE |
| main.step( "Comparing MN topology to ONOS topology" ) |
| compareRetry = 0 |
| while compareRetry < 3: |
| #While loop for retry |
| devices = main.topo.getAllDevices( main ) |
| ports = main.topo.getAllPorts( main ) |
| links = main.topo.getAllLinks( main) |
| mnSwitches = main.Mininet1.getSwitches() |
| mnLinks = main.Mininet1.getLinks(timeout=180) |
| |
| for controller in range(len(main.activeNodes)): |
| # controllerStr = str( main.activeNodes[controller] + 1 ) |
| if devices[ controller ] and ports[ controller ] and \ |
| "Error" not in devices[ controller ] and \ |
| "Error" not in ports[ controller ]: |
| currentDevicesResult = main.Mininet1.compareSwitches( |
| mnSwitches, |
| json.loads( devices[ controller ] ), |
| json.loads( ports[ controller ] ) ) |
| else: |
| currentDevicesResult = main.FALSE |
| |
| if links[ controller ] and "Error" not in links[ controller ]: |
| currentLinksResult = main.Mininet1.compareLinks( |
| mnSwitches, mnLinks, |
| json.loads( links[ controller ] ) ) |
| else: |
| currentLinksResult = main.FALSE |
| |
| stepResult = stepResult and currentDevicesResult and currentLinksResult |
| if stepResult: |
| break |
| compareRetry += 1 |
| utilities.assert_equals(expect=main.TRUE, |
| actual=stepResult, |
| onpass=" Topology match Mininet", |
| onfail="ONOS Topology doesn't match Mininet") |
| |
| if stepResult: |
| if main.hostDiscover: |
| hostList = [] |
| for i in range( 1, int( main.currScale ) + 1 ): |
| for j in range( 1, int( main.currScale ) + 1) : |
| # Generate host list |
| hoststr = "h" + str(i) + "x" + str(j) |
| hostList.append(hoststr) |
| for i in range( len(hostList) ): |
| totalHost = main.topo.sendArpPackage( main, hostList[i] ) |
| time.sleep( main.hostDiscoverSleep ) |
| if totalHost < 0: |
| # if totalHost less than 0 which means dependence function has exception. |
| main.log.info( "Error when discover host!" ) |
| break |
| if totalHost == int( main.currScale ) * int( main.currScale ): |
| main.log.info( "Discovered all hosts" ) |
| stepResult = stepResult and main.TRUE |
| else: |
| main.log.warn( "Some hosts ware not discovered by ONOS... Topology doesn't match!" ) |
| stepResult = main.FALSE |
| utilities.assert_equals(expect=main.TRUE, |
| actual=stepResult, |
| onpass=" Topology match Mininet", |
| onfail="ONOS Topology doesn't match Mininet") |
| main.log.info( "Finished this iteration, continue to scale next topology." ) |
| else: |
| main.log.info( "Clean up and exit TestON. Finished this test." ) |
| main.cleanup() |
| main.exit() |
| |
| def CASE100( self, main ): |
| ''' |
| Bring Down node 3 |
| ''' |
| |
| main.case("Bring ONOS node 3 down: TORUS %sx%s" % (main.currScale, main.currScale)) |
| main.caseExplanation = "Balance masters to make sure " +\ |
| "each controller has some devices and " +\ |
| "stop ONOS node 3 service. " |
| |
| stepResult = main.FALSE |
| main.step( "Bringing down node 3" ) |
| # Always bring down the third node |
| main.deadNode = 2 |
| # Printing purposes |
| node = main.deadNode + 1 |
| main.log.info( "Stopping node %s" % node ) |
| stepResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] ) |
| main.log.info( "Removing dead node from list of active nodes" ) |
| main.activeNodes.pop( main.deadNode ) |
| |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Successfully bring down node 3", |
| onfail="Failed to bring down node 3" ) |
| |
| def CASE200( self, main ): |
| ''' |
| Bring up onos node |
| ''' |
| |
| main.case("Bring ONOS node 3 up: TORUS %sx%s" % (main.currScale, main.currScale)) |
| main.caseExplanation = "Bring node 3 back up and balance the masters" |
| |
| node = main.deadNode + 1 |
| main.log.info( "Starting node %s" % node ) |
| stepResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] ) |
| main.log.info( "Starting onos cli" ) |
| stepResult = stepResult and main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] ) |
| |
| main.log.info( "Adding previously dead node to list of active nodes" ) |
| main.activeNodes.append( main.deadNode ) |
| |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Successfully brought up onos node %s" % node, |
| onfail="Failed to bring up onos node %s" % node ) |
| |
| |
| time.sleep(main.nodeSleep) |
| |
| def CASE300( self, main ): |
| ''' |
| |
| Balancing Masters |
| ''' |
| time.sleep(main.balanceSleep) |
| main.step( "Balancing Masters" ) |
| |
| stepResult = main.FALSE |
| if main.activeNodes: |
| controller = main.activeNodes[0] |
| stepResult = utilities.retry( main.CLIs[controller].balanceMasters, |
| main.FALSE, |
| [], |
| sleep=3, |
| attempts=3 ) |
| |
| else: |
| main.log.error( "List of active nodes is empty" ) |
| utilities.assert_equals( expect=main.TRUE, |
| actual=stepResult, |
| onpass="Balance masters was successfull", |
| onfail="Failed to balance masters") |
| time.sleep(main.balanceSleep) |
| |
| def CASE1000( self, main ): |
| ''' |
| Report errors/warnings/exceptions |
| ''' |
| # Compare the slowest Node through total time of each node |
| slowestNode = 0 |
| slowestTotalTime = 0 |
| # Second capture |
| for i in range( 3 ): |
| # Calculate total time |
| main.allinfo[ 1 ][ 'info' + str( i )][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'end' ], 'last', index=i, funcMode='TD' ) |
| # Compare the total time |
| if main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ] > slowestTotalTime: |
| slowestTotalTime = main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ] |
| slowestNode = i |
| # Calculate switch connection time |
| main.allinfo[ 1 ][ 'info' + str( i )][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'start' ], 'last', index=i, funcMode='TD' ) |
| # Calculate the time from last switch connection to the end |
| main.allinfo[ 1 ][ 'info' + str( i )][ 'lastSwToTopology' ] = main.allinfo[ 1 ][ |
| 'info' + str( i )][ 'totalTime' ] - main.allinfo[ 1 ][ 'info' + str( i )][ |
| 'swConnection' ] |
| # Calculate the disconnecti rate |
| main.allinfo[ 1 ][ 'info' + str( i )][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'Disconnect' ], 'num', main.searchTerm[ 'start' ],'num', index=i, funcMode='DR' ) |
| |
| if ( main.allinfo[ 0 ] != main.allinfo[ 1 ] ): |
| main.log.error( "The results of two capture are different!" ) |
| main.log.debug( "The data is " + str( main.allinfo ) ) |
| if main.writeData != -1: |
| main.log.info( "Write the date into database" ) |
| # write the date into data base |
| with open( main.dbFilePath, "a" ) as dbFile: |
| temp = str( main.currScale ) |
| temp += ",'baremetal1'" |
| # put result from second capture into data base |
| temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'totalTime' ] ) |
| temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'swConnection' ] ) |
| temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'lastSwToTopology' ] ) |
| temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'disconnectRate' ] ) |
| temp += "\n" |
| dbFile.write( temp ) |
| else: |
| main.log.error( "The data from log is wrong!" ) |
| main.writeData = 1 |
| main.case( "Checking logs for errors, warnings, and exceptions" ) |
| main.log.info( "Error report: \n" ) |
| main.ONOSbench.logReport( main.ONOSip[ 0 ], |
| [ "INFO", |
| "FOLLOWER", |
| "WARN", |
| "flow", |
| "ERROR", |
| "Except" ], |
| "s" ) |