TestON/tests/SCPFscaleTopo/SCPFscaleTopo.py

# Testing network scalability, this test suite scales up a network topology
# using mininet and verifies ONOS stability

class SCPFscaleTopo:

    def __init__( self ):
        self.default = ''

    def CASE1( self, main ):
        import time
        import os
        import imp
        import re

        """
        - Construct tests variables
        - GIT ( optional )
            - Checkout ONOS master branch
            - Pull latest ONOS code
        - Building ONOS ( optional )
            - Install ONOS package
            - Build ONOS package
        """

        main.case( "Constructing test variables" )
        main.step( "Constructing test variables" )
        stepResult = main.FALSE

        main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
        main.apps = main.params[ 'ENV' ][ 'cellApps' ]
        gitBranch = main.params[ 'GIT' ][ 'branch' ]
        main.dependencyPath = main.testOnDirectory + \
                              main.params[ 'DEPENDENCY' ][ 'path' ]
        main.multiovs = main.params[ 'DEPENDENCY' ][ 'multiovs' ]
        main.topoName = main.params[ 'TOPOLOGY' ][ 'topology' ]
        main.numCtrls = int( main.params[ 'CTRL' ][ 'numCtrls' ] )
        main.topoScale = ( main.params[ 'TOPOLOGY' ][ 'scale' ] ).split( "," )
        main.topoScaleSize = len( main.topoScale )
        wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
        wrapperFile2 = main.params[ 'DEPENDENCY' ][ 'wrapper2' ]
        wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
        main.topoCmpAttempts = int( main.params[ 'ATTEMPTS' ][ 'topoCmp' ] )
        main.pingallAttempts = int( main.params[ 'ATTEMPTS' ][ 'pingall' ] )
        main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
        main.fwdSleep = int( main.params[ 'SLEEP' ][ 'fwd' ] )
        main.balanceSleep = int( main.params[ 'SLEEP' ][ 'balance' ] )
        main.nodeDownSleep = int( main.params[ 'SLEEP' ][ 'nodeDown' ] )
        main.nodeUpSleep = int( main.params[ 'SLEEP' ][ 'nodeUp' ] )
        main.pingallSleep = int( main.params[ 'SLEEP' ][ 'pingall' ] )
        main.stopMNSleep = int( main.params[ 'SLEEP' ][ 'stopMN' ] )
        main.startMNSleep = int( main.params[ 'SLEEP' ][ 'startMN' ] )
        main.pingTimeout = int( main.params[ 'TIMEOUT' ][ 'pingall' ] )
        gitPull = main.params[ 'GIT' ][ 'pull' ]
        main.homeDir = os.path.expanduser('~')
        main.cellData = {} # for creating cell file
        main.hostsData = {}
        main.CLIs = []
        main.ONOSip = []
        main.activeNodes = []
        main.ONOSip = main.ONOSbench.getOnosIps()

        for i in range(main.numCtrls):
                main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )

        main.startUp = imp.load_source( wrapperFile1,
                                        main.dependencyPath +
                                        wrapperFile1 +
                                        ".py" )

        main.scaleTopoFunction = imp.load_source( wrapperFile2,
                                                  main.dependencyPath +
                                                  wrapperFile2 +
                                                  ".py" )

        main.topo = imp.load_source( wrapperFile3,
                                     main.dependencyPath +
                                     wrapperFile3 +
                                     ".py" )

        main.ONOSbench.scp( main.Mininet1,
                            main.dependencyPath +
                            main.multiovs,
                            main.Mininet1.home,
                            direction="to" )

        if main.CLIs:
                stepResult = main.TRUE
        else:
            main.log.error( "Did not properly created list of " +
                            "ONOS CLI handle" )
            stepResult = main.FALSE

        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully construct " +
                                        "test variables ",
                                 onfail="Failed to construct test variables" )

        if gitPull == 'True':
            main.step( "Building ONOS in " + gitBranch + " branch" )
            onosBuildResult = main.startUp.onosBuild( main, gitBranch )
            stepResult = onosBuildResult
            utilities.assert_equals( expect=main.TRUE,
                                     actual=stepResult,
                                     onpass="Successfully compiled " +
                                            "latest ONOS",
                                     onfail="Failed to compile " +
                                            "latest ONOS" )
        else:
            main.log.warn( "Did not pull new code so skipping mvn " +
                           "clean install" )


    def CASE2( self, main):
        """
        - Set up cell
            - Create cell file
            - Set cell file
            - Verify cell file
        - Kill ONOS process
        - Uninstall ONOS cluster
        - Verify ONOS start up
        - Install ONOS cluster
        - Connect to cli
        """

        main.case( "Starting up " + str( main.numCtrls ) +
                   " node(s) ONOS cluster" )
        main.caseExplanation = "Set up ONOS with " + str( main.numCtrls ) +\
                                " node(s) ONOS cluster"



        #kill off all onos processes
        main.log.info( "Safety check, killing all ONOS processes" +
                       " before initiating enviornment setup" )

        for i in range( main.numCtrls ):
            main.ONOSbench.onosDie( main.ONOSip[ i ] )

        tempOnosIp = []
        for i in range( main.numCtrls ):
            tempOnosIp.append( main.ONOSip[i] )

        main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
                                       "temp", main.Mininet1.ip_address,
                                       main.apps, tempOnosIp )

        main.step( "Apply cell to environment" )
        cellResult = main.ONOSbench.setCell( "temp" )
        verifyResult = main.ONOSbench.verifyCell()
        stepResult = cellResult and verifyResult
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully applied cell to " + \
                                        "environment",
                                 onfail="Failed to apply cell to environment " )

        main.step( "Creating ONOS package" )
        packageResult = main.ONOSbench.onosPackage()
        stepResult = packageResult
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully created ONOS package",
                                 onfail="Failed to create ONOS package" )

        time.sleep( main.startUpSleep )
        main.step( "Uninstalling ONOS package" )
        onosUninstallResult = main.TRUE
        for ip in main.ONOSip:
            onosUninstallResult = onosUninstallResult and \
                    main.ONOSbench.onosUninstall( nodeIp=ip )
        stepResult = onosUninstallResult
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully uninstalled ONOS package",
                                 onfail="Failed to uninstall ONOS package" )

        time.sleep( main.startUpSleep )
        main.step( "Installing ONOS package" )
        onosInstallResult = main.TRUE
        for i in range( main.numCtrls ):
            onosInstallResult = onosInstallResult and \
                    main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
        stepResult = onosInstallResult
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully installed ONOS package",
                                 onfail="Failed to install ONOS package" )

        time.sleep( main.startUpSleep )
        main.step( "Starting ONOS service" )
        stopResult = main.TRUE
        startResult = main.TRUE
        onosIsUp = main.TRUE

        for i in range( main.numCtrls ):
            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
        if onosIsUp == main.TRUE:
            main.log.report( "ONOS instance is up and ready" )
        else:
            main.log.report( "ONOS instance may not be up, stop and " +
                             "start ONOS again " )

            for i in range( main.numCtrls ):
                stopResult = stopResult and \
                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
            for i in range( main.numCtrls ):
                startResult = startResult and \
                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
        stepResult = onosIsUp and stopResult and startResult
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="ONOS service is ready",
                                 onfail="ONOS service did not start properly" )

        main.step( "Start ONOS cli" )
        cliResult = main.TRUE
        for i in range( main.numCtrls ):
            cliResult = cliResult and \
                        main.CLIs[ i ].startOnosCli( main.ONOSip[ i ] )
            main.activeNodes.append( i )
        stepResult = cliResult
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully start ONOS cli",
                                 onfail="Failed to start ONOS cli" )


    def CASE10( self, main ):
        """
            Starting up torus topology, pingall, and compare topo
        """
        import json

        main.case( "Starting up Mininet and verifying topology" )
        main.caseExplanation = "Starting Mininet with a scalling topology and " +\
                "comparing topology elements between Mininet and ONOS"

        main.log.info( "Checking if mininet is already running" )
        if len( main.topoScale ) < main.topoScaleSize:
            main.log.info( "Mininet is already running. Stopping mininet." )
            main.Mininet1.stopNet()
            time.sleep(main.stopMNSleep)
        else:
            main.log.info( "Mininet was not running" )

        if main.topoScale:
            scale = main.topoScale.pop(0)
        else: main.log.error( "topology scale is empty" )


        main.step( "Starting up TORUS %sx%s topology" % (scale, scale) )

        main.log.info( "Constructing Mininet command" )
        mnCmd = " mn --custom " + main.Mininet1.home + main.multiovs +\
                " --switch ovsm --topo " + main.topoName + ","+ scale + "," + scale

        for i in range( main.numCtrls ):
                mnCmd += " --controller remote,ip=" + main.ONOSip[ i ]

        stepResult = main.Mininet1.startNet(mnCmd=mnCmd)
        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass=main.topoName +
                                    " topology started successfully",
                                 onfail=main.topoName +
                                    " topology failed to start" )

        time.sleep( main.startMNSleep )

        main.step( "Pinging all hosts" )

        for i in range(main.pingallAttempts):
            pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
            if not pingResult:
                main.log.warn( "Pingall attempt: %s failed" % (i+1) )
                time.sleep(main.pingallSleep)
            else: break

        utilities.assert_equals( expect=main.TRUE,
                                 actual=pingResult,
                                 onpass="Pingall successfull",
                                 onfail="Pingall failed" )

        main.log.info( "Gathering topology information" )
        devicesResults = main.TRUE
        linksResults = main.TRUE
        hostsResults = main.TRUE

        devices = main.topo.getAllDevices( main )
        hosts = main.topo.getAllHosts( main )
        ports = main.topo.getAllPorts( main )
        links = main.topo.getAllLinks( main )
        clusters = main.topo.getAllClusters( main )

        mnSwitches = main.Mininet1.getSwitches()
        mnLinks = main.Mininet1.getLinks()
        mnHosts = main.Mininet1.getHosts()

        main.step( "Comparing MN topology to ONOS topology" )

        for controller in range(len(main.activeNodes)):
            controllerStr = str( main.activeNodes[controller] + 1 )
            if devices[ controller ] and ports[ controller ] and\
                "Error" not in devices[ controller ] and\
                "Error" not in ports[ controller ]:

                currentDevicesResult = main.Mininet1.compareSwitches(
                        mnSwitches,
                        json.loads( devices[ controller ] ),
                        json.loads( ports[ controller ] ) )
            else:
                currentDevicesResult = main.FALSE
            utilities.assert_equals( expect=main.TRUE,
                                     actual=currentDevicesResult,
                                     onpass="ONOS" + controllerStr +
                                     " Switches view is correct",
                                     onfail="ONOS" + controllerStr +
                                     " Switches view is incorrect" )

            if links[ controller ] and "Error" not in links[ controller ]:
                currentLinksResult = main.Mininet1.compareLinks(
                        mnSwitches, mnLinks,
                        json.loads( links[ controller ] ) )
            else:
                currentLinksResult = main.FALSE
            utilities.assert_equals( expect=main.TRUE,
                                     actual=currentLinksResult,
                                     onpass="ONOS" + controllerStr +
                                     " links view is correct",
                                     onfail="ONOS" + controllerStr +
                                     " links view is incorrect" )

            if hosts[ controller ] or "Error" not in hosts[ controller ]:
                currentHostsResult = main.Mininet1.compareHosts(
                        mnHosts,
                        json.loads( hosts[ controller ] ) )
            else:
                currentHostsResult = main.FALSE
        utilities.assert_equals( expect=main.TRUE,
                                 actual=currentHostsResult,
                                 onpass="ONOS" + controllerStr +
                                 " hosts exist in Mininet",
                                 onfail="ONOS" + controllerStr +
                                 " hosts don't match Mininet" )

    def CASE11( self, main ):
        """
            Pingall, and compare topo
        """
        import json

        scale = main.topoScale[0]

        main.case( "Verifying topology: TORUS %sx%s" % (scale, scale) )
        main.caseExplanation = "Pinging all hosts andcomparing topology " +\
                "elements between Mininet and ONOS"
        main.step( "Pinging all hosts" )

        for i in range(main.pingallAttempts):
            pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
            if not pingResult:
                main.log.warn( "Pingall attempt: %s failed" % (i+1) )
                time.sleep(main.pingallSleep)
            else: break

        utilities.assert_equals( expect=main.TRUE,
                                 actual=pingResult,
                                 onpass="Pingall successfull",
                                 onfail="Pingall failed" )

        main.log.info( "Gathering topology information" )
        devicesResults = main.TRUE
        linksResults = main.TRUE
        hostsResults = main.TRUE

        devices = main.topo.getAllDevices( main )
        hosts = main.topo.getAllHosts( main )
        ports = main.topo.getAllPorts( main )
        links = main.topo.getAllLinks( main )
        clusters = main.topo.getAllClusters( main )

        mnSwitches = main.Mininet1.getSwitches()
        mnLinks = main.Mininet1.getLinks()
        mnHosts = main.Mininet1.getHosts()

        main.step( "Comparing MN topology to ONOS topology" )

        for controller in range(len(main.activeNodes)):
            controllerStr = str( main.activeNodes[controller] + 1 )
            if devices[ controller ] and ports[ controller ] and\
                "Error" not in devices[ controller ] and\
                "Error" not in ports[ controller ]:

                currentDevicesResult = main.Mininet1.compareSwitches(
                        mnSwitches,
                        json.loads( devices[ controller ] ),
                        json.loads( ports[ controller ] ) )
            else:
                currentDevicesResult = main.FALSE
            utilities.assert_equals( expect=main.TRUE,
                                     actual=currentDevicesResult,
                                     onpass="ONOS" + controllerStr +
                                     " Switches view is correct",
                                     onfail="ONOS" + controllerStr +
                                     " Switches view is incorrect" )

            if links[ controller ] and "Error" not in links[ controller ]:
                currentLinksResult = main.Mininet1.compareLinks(
                        mnSwitches, mnLinks,
                        json.loads( links[ controller ] ) )
            else:
                currentLinksResult = main.FALSE
            utilities.assert_equals( expect=main.TRUE,
                                     actual=currentLinksResult,
                                     onpass="ONOS" + controllerStr +
                                     " links view is correct",
                                     onfail="ONOS" + controllerStr +
                                     " links view is incorrect" )

            if hosts[ controller ] or "Error" not in hosts[ controller ]:
                currentHostsResult = main.Mininet1.compareHosts(
                        mnHosts,
                        json.loads( hosts[ controller ] ) )
            else:
                currentHostsResult = main.FALSE
        utilities.assert_equals( expect=main.TRUE,
                                 actual=currentHostsResult,
                                 onpass="ONOS" + controllerStr +
                                 " hosts exist in Mininet",
                                 onfail="ONOS" + controllerStr +
                                 " hosts don't match Mininet" )


    def CASE100( self, main ):
        '''
            Balance masters, ping and bring third ONOS node down
        '''
        scale = main.topoScale[0]

        main.case("Balancing Masters and bring ONOS node 3 down: TORUS %sx%s" % (scale, scale))
        main.caseExplanation = "Balance masters to make sure " +\
                        "each controller has some devices and " +\
                        "stop ONOS node 3 service. "

        main.step( "Balancing Masters" )
        stepResult = main.FALSE
        if main.activeNodes:
            controller = main.activeNodes[0]
            stepResult = main.CLIs[controller].balanceMasters()
        else: main.log.error( "List of active nodes is empty" )

        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Balance masters was successfull",
                                 onfail="Failed to balance masters")

        time.sleep(main.balanceSleep)

        main.step( "Pinging all hosts" )

        for i in range(main.pingallAttempts):
            pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
            if not pingResult:
                main.log.warn( "Pingall attempt: %s failed" % (i+1) )
                time.sleep(main.pingallSleep)
            else: break

        utilities.assert_equals( expect=main.TRUE,
                                 actual=pingResult,
                                 onpass="Pingall successfull",
                                 onfail="Pingall failed" )

        main.step( "Bringing down node 3" )

        # Always bring down the third node
        main.deadNode = 2

        # Printing purposes
        node = main.deadNode + 1

        main.log.info( "Stopping node %s" % node )
        stepResult = main.ONOSbench.onosStop( main.ONOSip[ main.deadNode ] )

        main.log.info( "Removing dead node from list of active nodes" )
        main.activeNodes.pop( main.deadNode )

        utilities.assert_equals( expect=main.TRUE,
                             actual=stepResult,
                             onpass="Successfully brought down onos node %s" % node,
                             onfail="Failed to bring down onos node %s" % node )

        time.sleep(main.nodeDownSleep)


    def CASE200( self, main ):
        '''
            Bring up onos node and balance masters
        '''

        scale = main.topoScale[0]

        main.case("Bring ONOS node 3 up and balance masters: TORUS %sx%s" % (scale, scale))
        main.caseExplanation = "Bring node 3 back up and balance the masters"

        node = main.deadNode + 1

        main.log.info( "Starting node %s" % node )
        stepResult = main.ONOSbench.onosStart( main.ONOSip[ main.deadNode ] )

        main.log.info( "Starting onos cli" )
        stepResult = stepResult and main.CLIs[ main.deadNode ].startOnosCli( main.ONOSip[ main.deadNode ] )

        main.log.info( "Adding previously dead node to list of active nodes" )
        main.activeNodes.append( main.deadNode )

        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Successfully brought up onos node %s" % node,
                                 onfail="Failed to bring up onos node %s" % node )


        time.sleep(main.nodeUpSleep)

        main.step( "Balancing Masters" )
        stepResult = main.FALSE
        if main.activeNodes:
            controller = main.activeNodes[0]
            stepResult = main.CLIs[controller].balanceMasters()
        else: main.log.error( "List of active nodes is empty" )

        utilities.assert_equals( expect=main.TRUE,
                                 actual=stepResult,
                                 onpass="Balance masters was successfull",
                                 onfail="Failed to balance masters")

        time.sleep(main.balanceSleep)

        for i in range(main.pingallAttempts):
            pingResult = main.Mininet1.pingall(timeout=main.pingTimeout)
            if not pingResult:
                main.log.warn( "Pingall attempt: %s failed" % (i+1) )
                time.sleep(main.pingallSleep)
            else: break

    def CASE1000( self, main ):
        '''
            Report errors/warnings/exceptions
        '''
        main.case( "Checking logs for errors, warnings, and exceptions" )
        main.log.info("Error report: \n" )
        main.ONOSbench.logReport( main.ONOSip[ 0 ],
                                                            [ "INFO",
                                                              "FOLLOWER",
                                                              "WARN",
                                                              "flow",
                                                              "ERROR",
                                                              "Except" ],
                                                            "s" )