TestON/tests/SCPF/SCPFintentRerouteLat/SCPFintentRerouteLat.py

# SCPFintentRerouteLat
"""
SCPFintentRerouteLat
    - Test Intent Reroute Latency
    - Test Algorithm:
        1. Start Null Provider reroute Topology
        2. Using Push-test-intents to push batch size intents from switch 1 to switch 7
        3. Cut the link between switch 3 and switch 4 (the path will reroute to switch 8)
        4. Get the topology time stamp
        5. Get Intent reroute(Installed) time stamp from each nodes
        6. Use the latest intent time stamp subtract topology time stamp
    - This test will run 5 warm up by default, warm up iteration can be setup in Param file
    - The intent batch size will default set to 1, 100, and 1000, also can be set in Param file
    - The unit of the latency result is milliseconds
"""


class SCPFintentRerouteLat:
    def __init__(self):
        self.default = ''

    def CASE0( self, main ):
        import imp
        import os
        '''
        - GIT
        - BUILDING ONOS
            Pull specific ONOS branch, then Build ONOS ono ONOS Bench.
            This step is usually skipped. Because in a Jenkins driven automated
            test env. We want Jenkins jobs to pull&build for flexibility to handle
            different versions of ONOS.
        - Construct tests variables
        '''
        gitPull = main.params['GIT']['gitPull']
        gitBranch = main.params['GIT']['gitBranch']

        main.case("Pull onos branch and build onos on Teststation.")

        if gitPull == 'True':
            main.step("Git Checkout ONOS branch: " + gitBranch)
            stepResult = main.ONOSbench.gitCheckout(branch=gitBranch)
            utilities.assert_equals(expect=main.TRUE,
                                    actual=stepResult,
                                    onpass="Successfully checkout onos branch.",
                                    onfail="Failed to checkout onos branch. Exiting test...")
            if not stepResult:
                main.exit()

            main.step("Git Pull on ONOS branch:" + gitBranch)
            stepResult = main.ONOSbench.gitPull()
            utilities.assert_equals(expect=main.TRUE,
                                    actual=stepResult,
                                    onpass="Successfully pull onos. ",
                                    onfail="Failed to pull onos. Exiting test ...")
            if not stepResult: main.exit()

            main.step("Building ONOS branch: " + gitBranch)
            stepResult = main.ONOSbench.cleanInstall(skipTest=True)
            utilities.assert_equals(expect=main.TRUE,
                                    actual=stepResult,
                                    onpass="Successfully build onos.",
                                    onfail="Failed to build onos. Exiting test...")
            if not stepResult:
                main.exit()

        else:
            main.log.warn("Skipped pulling onos and Skipped building ONOS")
        main.onosIp = main.ONOSbench.getOnosIps()
        main.apps = main.params['ENV']['cellApps']
        main.BENCHUser = main.params['BENCH']['user']
        main.BENCHIp = main.params['BENCH']['ip1']
        main.MN1Ip = main.params['MN']['ip1']
        main.maxNodes = int(main.params['max'])
        main.skipMvn = main.params['TEST']['skipCleanInstall']
        main.cellName = main.params['ENV']['cellName']
        main.scale = (main.params['SCALE']).split(",")
        main.timeout = int(main.params['SLEEP']['timeout'])
        main.startUpSleep = int(main.params['SLEEP']['startup'])
        main.installSleep = int(main.params['SLEEP']['install'])
        main.verifySleep = int(main.params['SLEEP']['verify'])
        main.setMasterSleep = int(main.params['SLEEP']['setmaster'])
        main.verifyAttempts = int(main.params['ATTEMPTS']['verify'])
        main.sampleSize = int(main.params['TEST']['sampleSize'])
        main.warmUp = int(main.params['TEST']['warmUp'])
        main.ingress = main.params['TEST']['ingress']
        main.egress = main.params['TEST']['egress']
        main.debug = main.params['TEST']['debug']
        main.flowObj = main.params['TEST']['flowObj']
        main.deviceCount = int(main.params['TEST']['deviceCount'])
        main.end1 = main.params['TEST']['end1']
        main.end2 = main.params['TEST']['end2']
        main.searchTerm = main.params['SEARCHTERM']
        if main.flowObj == "True":
            main.flowObj = True
            main.dbFileName = main.params['DATABASE']['dbFlowObj']
            main.intentsList = (main.params['TEST']['FObjintents']).split(",")
        else:
            main.flowObj = False
            main.dbFileName = main.params['DATABASE']['dbName']
            main.intentsList = (main.params['TEST']['intents']).split(",")

        for i in range(0, len(main.intentsList)):
            main.intentsList[i] = int(main.intentsList[i])
            # Create DataBase file
        main.log.info("Create Database file " + main.dbFileName)
        resultsDB = open(main.dbFileName, "w+")
        resultsDB.close()
        file1 = main.params[ "DEPENDENCY" ][ "FILE1" ]
        main.dependencyPath = os.path.dirname( os.getcwd() ) + main.params[ "DEPENDENCY" ][ "PATH" ]
        main.intentRerouteLatFuncs = imp.load_source(file1, main.dependencyPath + file1 + ".py")

        main.record = 0

    def CASE1( self, main ):
        '''
            clean up test environment and set up
        '''
        import time

        main.log.info("Get ONOS cluster IP")
        print(main.scale)
        main.numCtrls = int(main.scale[0])
        main.ONOSip = []
        main.maxNumBatch = 0
        main.AllONOSip = main.ONOSbench.getOnosIps()
        for i in range(main.numCtrls):
            main.ONOSip.append(main.AllONOSip[i])
        main.log.info(main.ONOSip)
        main.CLIs = []
        main.log.info("Creating list of ONOS cli handles")
        for i in range(main.numCtrls):
            main.CLIs.append(getattr(main, 'ONOS%scli' % (i + 1)))

        if not main.CLIs:
            main.log.error("Failed to create the list of ONOS cli handles")
            main.cleanup()
            main.exit()

        main.commit = main.ONOSbench.getVersion(report=True)
        main.commit = main.commit.split(" ")[1]
        main.log.info("Starting up %s node(s) ONOS cluster" % main.numCtrls)
        main.log.info("Safety check, killing all ONOS processes" +
                      " before initiating environment setup")

        for i in range(main.numCtrls):
            main.ONOSbench.onosStop(main.ONOSip[i])
            main.ONOSbench.onosKill(main.ONOSip[i])

        main.log.info("NODE COUNT = %s" % main.numCtrls)
        main.ONOSbench.createCellFile(main.ONOSbench.ip_address,
                                      main.cellName,
                                      main.MN1Ip,
                                      main.apps,
                                      main.ONOSip)
        main.step("Apply cell to environment")
        cellResult = main.ONOSbench.setCell(main.cellName)
        verifyResult = main.ONOSbench.verifyCell()
        stepResult = cellResult and verifyResult
        utilities.assert_equals(expect=main.TRUE,
                                actual=stepResult,
                                onpass="Successfully applied cell to " + \
                                       "environment",
                                onfail="Failed to apply cell to environment ")

        main.step("Creating ONOS package")
        packageResult = main.ONOSbench.buckBuild()
        stepResult = packageResult
        utilities.assert_equals(expect=main.TRUE,
                                actual=stepResult,
                                onpass="Successfully created ONOS package",
                                onfail="Failed to create ONOS package")

        main.step("Uninstall ONOS package on all Nodes")
        uninstallResult = main.TRUE
        for i in range(int(main.numCtrls)):
            main.log.info("Uninstalling package on ONOS Node IP: " + main.ONOSip[i])
            u_result = main.ONOSbench.onosUninstall(main.ONOSip[i])
            utilities.assert_equals(expect=main.TRUE, actual=u_result,
                                    onpass="Test step PASS",
                                    onfail="Test step FAIL")
            uninstallResult = (uninstallResult and u_result)

        main.step("Install ONOS package on all Nodes")
        installResult = main.TRUE
        for i in range(int(main.numCtrls)):
            main.log.info("Installing package on ONOS Node IP: " + main.ONOSip[i])
            i_result = main.ONOSbench.onosInstall(node=main.ONOSip[i])
            utilities.assert_equals(expect=main.TRUE, actual=i_result,
                                    onpass="Test step PASS",
                                    onfail="Test step FAIL")
            installResult = installResult and i_result

        main.step( "Set up ONOS secure SSH" )
        secureSshResult = main.TRUE
        for i in range( int( main.numCtrls ) ):
            secureSshResult = secureSshResult and main.ONOSbench.onosSecureSSH( node=main.ONOSip[i] )
            utilities.assert_equals( expect=main.TRUE, actual=secureSshResult,
                                    onpass="Test step PASS",
                                    onfail="Test step FAIL" )

        main.step( "Starting ONOS service" )
        stopResult = main.TRUE
        startResult = main.TRUE
        onosIsUp = main.TRUE

        for i in range( main.numCtrls ):
            onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
        if onosIsUp == main.TRUE:
            main.log.report( "ONOS instance is up and ready" )
        else:
            main.log.report( "ONOS instance may not be up, stop and " +
                             "start ONOS again " )

            for i in range( main.numCtrls ):
                stopResult = stopResult and \
                        main.ONOSbench.onosStop( main.ONOSip[ i ] )
            for i in range( main.numCtrls ):
                startResult = startResult and \
                        main.ONOSbench.onosStart( main.ONOSip[ i ] )
        stepResult = onosIsUp and stopResult and startResult
        utilities.assert_equals( expect=main.TRUE, actual=stepResult,
                                 onpass="Test step PASS",
                                 onfail="Test step FAIL" )

        time.sleep(2)
        main.step("Start ONOS CLI on all nodes")
        cliResult = main.TRUE
        main.step(" Start ONOS cli using thread ")
        startCliResult = main.TRUE
        pool = []
        main.threadID = 0
        for i in range(int(main.numCtrls)):
            t = main.Thread(target=main.CLIs[i].startOnosCli,
                            threadID=main.threadID,
                            name="startOnosCli",
                            args=[main.ONOSip[i]],
                            kwargs={"onosStartTimeout": main.timeout})
            pool.append(t)
            t.start()
            main.threadID = main.threadID + 1
        for t in pool:
            t.join()
            startCliResult = startCliResult and t.result
        time.sleep(main.startUpSleep)

        # configure apps
        main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "deviceCount", value=main.deviceCount)
        main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "topoShape", value="reroute")
        main.CLIs[0].setCfg("org.onosproject.provider.nil.NullProviders", "enabled", value="true")
        main.CLIs[0].setCfg("org.onosproject.store.flow.impl.DistributedFlowRuleStore", "backupEnabled", value="false")
        if main.flowObj:
            main.CLIs[0].setCfg("org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator",
                                "useFlowObjectives", value="true")
        time.sleep( main.startUpSleep )
        for i in range( int( main.numCtrls ) ):
            main.CLIs[i].logSet( "DEBUG", "org.onosproject.metrics.topology" )
            main.CLIs[i].logSet( "DEBUG", "org.onosproject.metrics.intent" )
        # Balance Master
        main.CLIs[0].balanceMasters()
        if len(main.ONOSip) > 1:
            main.CLIs[0].deviceRole(main.end1[ 'name' ], main.ONOSip[0])
            main.CLIs[0].deviceRole(main.end2[ 'name' ], main.ONOSip[0])
        time.sleep( main.setMasterSleep )

    def CASE2( self, main ):
        import time
        import numpy
        import datetime
        import json
        # from scipy import stats

        ts = time.time()
        print(main.intentsList)
        for batchSize in main.intentsList:
            main.log.report("Intent Batch size: " + str(batchSize) + "\n      ")
            main.LatencyList = []
            main.LatencyListTopoToFirstInstalled = []
            main.LatencyListFirstInstalledToLastInstalled = []
            main.validRun = 0
            main.invalidRun = 0
            # initial a variables to record the term of startLine in karaf logs of each node
            main.totalLines = []
            for i in range( main.numCtrls ):
                main.totalLines.append( '' )
            while main.validRun <= main.warmUp + main.sampleSize and main.invalidRun <= 20:
                if main.validRun >= main.warmUp:
                    main.log.info("================================================")
                    main.log.info("Starting test iteration: {} ".format( main.validRun - main.warmUp))
                    main.log.info("Total iteration: {}".format( main.validRun + main.invalidRun))
                    main.log.info("================================================")
                else:
                    main.log.info("====================Warm Up=====================")

                # push intents
                main.CLIs[0].pushTestIntents(main.ingress, main.egress, batchSize,
                                             offset=1, options="-i", timeout=main.timeout)

                # check links and flows
                k = 0
                main.verify = main.FALSE
                linkCheck = 0
                flowsCheck = 0
                while k <= main.verifyAttempts:
                    time.sleep(main.verifySleep)
                    summary = json.loads(main.CLIs[0].summary(timeout=main.timeout))
                    linkCheck = summary.get("links")
                    flowsCheck = summary.get("flows")
                    if linkCheck == main.deviceCount * 2 and flowsCheck == batchSize * (main.deviceCount - 1 ):
                        main.log.info("links: {}, flows: {} ".format(linkCheck, flowsCheck))
                        main.verify = main.TRUE
                        break
                    k += 1
                if not main.verify:
                    main.log.warn("Links or flows number are not match!")
                    main.log.warn("links: {}, flows: {} ".format(linkCheck, flowsCheck))
                    # bring back topology
                    main.intentRerouteLatFuncs.bringBackTopology( main )
                    if main.validRun >= main.warmUp:
                        main.invalidRun += 1
                        continue
                    else:
                        main.validRun += 1
                        continue
                # Bring link down
                main.CLIs[0].link( main.end1[ 'port' ], main.end2[ 'port' ], "down",
                                  timeout=main.timeout, showResponse=False)
                main.verify = main.FALSE
                k = 0
                topoManagerLog = ""
                while k <= main.verifyAttempts:
                    time.sleep(main.verifySleep)
                    summary = json.loads(main.CLIs[0].summary(timeout=main.timeout))
                    linkCheck = summary.get("links")
                    flowsCheck = summary.get("flows")
                    if linkCheck == (main.deviceCount - 1) * 2:
                        main.log.info("links: {}, flows: {} ".format(linkCheck, flowsCheck))
                        main.verify = main.TRUE
                        break
                    k += 1
                if not main.verify:
                    main.log.warn("Links number are not match in TopologyManager log!")
                    main.log.warn(topoManagerLog)
                    # bring back topology
                    main.intentRerouteLatFuncs.bringBackTopology( main )
                    if main.validRun >= main.warmUp:
                        main.invalidRun += 1
                        continue
                    else:
                        main.validRun += 1
                        continue
                # record the link romving time as the startLine
                for i in range( main.numCtrls ):
                    logNum = main.intentRerouteLatFuncs.checkLog( main, i )
                    main.totalLines[i] = str(main.CLIs[ i ].getTimeStampFromLog( "last", "LINK_REMOVED", "time = ", " ", logNum=logNum ))
                    main.log.info("Node " + str( i+1 ) + ": the start timestamp is " + main.totalLines[i] + " this iteration" )
                #Calculate values
                lastTopologyToFirstInstalled, firstInstalledToLastInstalled, totalTime = main.intentRerouteLatFuncs.getValues( main )
                if totalTime == -1:
                    if main.validRun >= main.warmUp:
                        main.invalidRun += 1
                    else:
                        main.validRun += 1
                    continue
                else:
                    main.log.info("Get valid latency")
                    main.validRun += 1

                # Verify Summary after we bring up link, and withdrawn intents
                main.CLIs[0].link( main.end1[ 'port' ], main.end2[ 'port' ], "up",
                                  timeout=main.timeout)
                k = 0
                main.verify = main.FALSE
                linkCheck = 0
                flowsCheck = 0
                while k <= main.verifyAttempts:
                    time.sleep(main.verifySleep)
                    main.CLIs[0].removeAllIntents(purge=True, sync=True, timeout=main.timeout)
                    time.sleep(1)
                    main.CLIs[0].purgeWithdrawnIntents()
                    summary = json.loads(main.CLIs[0].summary())
                    linkCheck = summary.get("links")
                    flowsCheck = summary.get("flows")
                    intentCheck = summary.get("intents")
                    if linkCheck == main.deviceCount * 2 and flowsCheck == 0 and intentCheck == 0:
                        main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
                        main.verify = main.TRUE
                        break
                    k += 1
                if not main.verify:
                    main.log.error("links, flows, or intents are not correct!")
                    main.log.info("links: {}, flows: {}, intents: {} ".format(linkCheck, flowsCheck, intentCheck))
                    # bring back topology
                    main.intentRerouteLatFuncs.bringBackTopology( main )
                    continue
                main.log.info("total negative results num: " + str( main.record ) )

            aveLatency = 0
            aveLatencyTopoToFirstInstalled = 0
            aveLatencyFirstInstalledToLastInstalled = 0

            stdLatency = 0
            stdLatencyTopoToFirstInstalled = 0
            stdLatencyFirstInstalledToLastInstalled = 0

            aveLatency = numpy.average( main.LatencyList )
            aveLatencyTopoToFirstInstalled = numpy.average( main.LatencyListTopoToFirstInstalled )
            aveLatencyFirstInstalledToLastInstalled = numpy.average( main.LatencyListFirstInstalledToLastInstalled )

            stdLatency = numpy.std( main.LatencyList )
            stdLatencyTopoToFirstInstalled = numpy.std( main.LatencyListTopoToFirstInstalled )
            stdLatencyFirstInstalledToLastInstalled = numpy.std( main.LatencyListFirstInstalledToLastInstalled )

            main.log.report( "Scale: " + str( main.numCtrls ) + "  \tIntent batch: " + str( batchSize ) )
            main.log.report( "Total Latency average:................" + str( aveLatency ) )
            main.log.report( "Latency standard deviation:..........." + str( stdLatency ) )
            main.log.report( "Last Topology to first installed Latency average:................." + str( aveLatencyTopoToFirstInstalled ) )
            main.log.report( "Last Topology to first installed Latency standard deviation:......" + str( stdLatencyTopoToFirstInstalled ) )
            main.log.report( "First installed to last installed Latency average:................" + str( aveLatencyFirstInstalledToLastInstalled ) )
            main.log.report( "First installed to last installed Latency standard deviation:....." + str( stdLatencyFirstInstalledToLastInstalled ) )
            main.log.report( "________________________________________________________" )

            if not (numpy.isnan(aveLatency) or numpy.isnan(stdLatency)):
                # check if got NaN for result
                resultsDB = open(main.dbFileName, "a")
                resultsDB.write("'" + main.commit + "',")
                resultsDB.write(str(main.numCtrls) + ",")
                resultsDB.write(str(batchSize) + ",")
                resultsDB.write(str(aveLatency) + ",")
                resultsDB.write(str(stdLatency) + ",")
                resultsDB.write(str(aveLatencyTopoToFirstInstalled) + ",")
                resultsDB.write(str(stdLatencyTopoToFirstInstalled) + ",")
                resultsDB.write(str(aveLatencyFirstInstalledToLastInstalled) + ",")
                resultsDB.write(str(stdLatencyFirstInstalledToLastInstalled) + "\n")
                resultsDB.close()
        del main.scale[0]