TestON/tests/IntentEventTP/IntentEventTP.py

# ScaleOutTemplate
#
# CASE1 starts number of nodes specified in param file
#
# cameron@onlab.us

import sys
import os.path


class IntentEventTP:

    def __init__( self ):
        self.default = ''

    def CASE1( self, main ):           
                                        
        import time                     
        import os.path

        global init       
        try: 
            if type(init) is not bool: 
                init = False  
        except NameError: 
            init = False 
       
        #Load values from params file
        checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
        gitPull = main.params[ 'GIT' ][ 'autopull' ]
        cellName = main.params[ 'ENV' ][ 'cellName' ]
        Apps = main.params[ 'ENV' ][ 'cellApps' ]
        BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
        BENCHUser = main.params[ 'BENCH' ][ 'user' ]
        MN1Ip = main.params[ 'MN' ][ 'ip1' ]
        maxNodes = int(main.params[ 'availableNodes' ])
        skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
        cellName = main.params[ 'ENV' ][ 'cellName' ]        
        numSwitches = (main.params[ 'TEST' ][ 'numSwitches' ]).split(",")
        flowRuleBU = main.params[ 'TEST' ][ 'flowRuleBUEnabled' ]
        onBaremetal = main.params['isOnBaremetal']
        homeDir = os.path.expanduser('~')

        main.exceptions = [0]*11
        main.warnings = [0]*11
        main.errors = [0]*11

        # -- INIT SECTION, ONLY RUNS ONCE -- # 
        if init == False: 
            init = True
            global clusterCount             #number of nodes running
            global ONOSIp                   #list of ONOS IP addresses
            global scale 
            global commit

            clusterCount = 0
            ONOSIp = [ 0 ]
            scale = (main.params[ 'SCALE' ]).split(",")            
            clusterCount = int(scale[0])

            #Populate ONOSIp with ips from params 
            for i in range(1, maxNodes + 1): 
                ipString = 'ip' + str(i) 
                ONOSIp.append(main.params[ 'CTRL' ][ ipString ])   
            
            #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
            if skipMvn != "yes":
                mvnResult = main.ONOSbench.cleanInstall()

            #git
            main.step( "Git checkout and pull " + checkoutBranch )
            if gitPull == 'on':
                checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
                pullResult = main.ONOSbench.gitPull()

            else:
                checkoutResult = main.TRUE
                pullResult = main.TRUE
                main.log.info( "Skipped git checkout and pull" )
        
            commit = main.ONOSbench.getVersion()
            commit = (commit.split(" "))[1]
        
            resultsDB = open("IntentEventTPDB", "w+")
            resultsDB.close()

        # -- END OF INIT SECTION --#
         
        clusterCount = int(scale[0])
        scale.remove(scale[0])       
        
        #kill off all onos processes 
        main.log.step("Safety check, killing all ONOS processes")
        main.log.step("before initiating enviornment setup")
        for node in range(1, maxNodes + 1):
            main.ONOSbench.onosDie(ONOSIp[node])
        
        #Uninstall everywhere
        main.log.step( "Cleaning Enviornment..." )
        for i in range(1, maxNodes + 1):
            main.log.info(" Uninstalling ONOS " + str(i) )
            main.ONOSbench.onosUninstall( ONOSIp[i] )
       
        #construct the cell file
        main.log.info("Creating cell file")
        cellIp = []
        for node in range (1, clusterCount + 1):
            cellIp.append(ONOSIp[node])

        main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), *cellIp)

        main.step( "Set Cell" )
        main.ONOSbench.setCell(cellName)

        myDistribution = []
        for node in range (1, clusterCount + 1):
            myDistribution.append(numSwitches[node-1])

        #main.ONOSbench.createLinkGraphFile( BENCHIp,cellIp,myDistribution)

        if onBaremetal == "True":
            filename = "/onos/tools/package/bin/onos-service"
            serviceConfig = open(homeDir + filename, 'w+')
            serviceConfig.write("#!/bin/bash\n ")
            serviceConfig.write("#------------------------------------- \n ")
            serviceConfig.write("# Starts ONOS Apache Karaf container\n ")
            serviceConfig.write("#------------------------------------- \n ")
            serviceConfig.write("#export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}\n ")
            serviceConfig.write("""export JAVA_OPTS="${JAVA_OPTS:--Xms8G -Xmx8G}" \n """)
            serviceConfig.write("")
            serviceConfig.write("ONOS_HOME=/opt/onos \n ")
            serviceConfig.write("")
            serviceConfig.write("[ -d $ONOS_HOME ] && cd $ONOS_HOME || ONOS_HOME=$(dirname $0)/..\n")
            serviceConfig.write("""${ONOS_HOME}/apache-karaf-$KARAF_VERSION/bin/karaf "$@" \n """)
            serviceConfig.close()
            main.log.info("Set /onos/tools/package/bin/onos-service with 8G Xms/Xmx Options.")
      
        main.step( "Creating ONOS package" )
        packageResult = main.ONOSbench.onosPackage()  

        main.step( "verify cells" )
        verifyCellResult = main.ONOSbench.verifyCell()
      
        main.log.report( "Initializeing " + str( clusterCount ) + " node cluster." )
        for node in range(1, clusterCount + 1):
            main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
            main.ONOSbench.onosInstall( ONOSIp[node])

        for node in range(1, clusterCount + 1):
            for i in range( 2 ):
                isup = main.ONOSbench.isup( ONOSIp[node] )
                if isup:
                    main.log.info("ONOS " + str(node) + " is up\n")
                    break
            if not isup:
                main.log.report( "ONOS " + str(node) + " didn't start!" )
        main.log.info("Startup sequence complete")

        time.sleep(20)


        while True: 
            main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders deviceCount """ + str(clusterCount*10) + """ " """)
            main.ONOSbench.handle.expect(":~")
            main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
            main.ONOSbench.handle.expect(":~")
            if ("value=" + str(clusterCount*10)) in main.ONOSbench.handle.before:
                main.log.info("Device count set")
                main.log.info("before" + main.ONOSbench.handle.before)
                break
            time.sleep(10)
            main.log.info("cfg set failure, retrying")
            main.log.info("before" + main.ONOSbench.handle.before)
        
        while True:
            main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.provider.nil.NullProviders topoShape linear" """)
            main.ONOSbench.handle.expect(":~")
            main.ONOSbench.handle.sendline("""onos $OC1 "cfg get org.onosproject.provider.nil.NullProviders" """)
            main.ONOSbench.handle.expect(":~")
            if ("value=linear") in main.ONOSbench.handle.before:
                main.log.info("Device count set")
                main.log.info("before" + main.ONOSbench.handle.before)
                break
            time.sleep(10)
            main.log.info("cfg set failure, retrying")
            main.log.info("before" + main.ONOSbench.handle.before)

        main.ONOSbench.handle.sendline("""onos $OC1 "cfg set org.onosproject.store.flow.impl.NewDistributedFlowRuleStore backupEnabled """ + flowRuleBU + """" """)
        main.ONOSbench.handle.expect(":~")
        main.ONOSbench.handle.sendline("""onos $OC1 "cfg get" """)
        main.ONOSbench.handle.expect(":~")
        main.log.info(main.ONOSbench.handle.before)

        time.sleep(10)
        main.ONOSbench.handle.sendline("""onos $OC1 "null-simulation start" """)
        main.ONOSbench.handle.expect(":~")
        print main.ONOSbench.handle.before
        time.sleep(10)
        main.ONOSbench.handle.sendline("""onos $OC1 "balance-masters" """)
        main.ONOSbench.handle.expect(":~")
        print main.ONOSbench.handle.before
 
        lastOutput = "--"
        origin = time.time()
        clockStarted = False
        while True:
            main.ONOSbench.handle.sendline("onos $OC1 summary")
            main.ONOSbench.handle.expect(":~")
            main.log.info("before" + main.ONOSbench.handle.before)
            clusterCheck = ((main.ONOSbench.handle.before).splitlines())[3]
            print("\nBefore: " + str(clusterCheck))
            if ("SCC(s)=1,") in clusterCheck:
                break
            if clusterCheck != lastOutput:
                sameOutput = False
            elif clusterCheck == lastOutput:
                if clockStarted == False:
                    start = time.time()
                    clockStarted = True
                if time.time() > (start + 10):
                    main.log.error("TIMEOUT EXCEEDED: Clusters have not converged, continuing anyway...")
                    break
            lastOutput = clusterCheck
            time.sleep(5)
        main.ONOSbench.onosErrorLog(ONOSIp[1])

    def CASE2( self, main ): 
        import time
        import json
        import string
        import csv
        import numpy
        import os.path

        global currentNeighbors
        neighbors = []

        try:
            currentNeighbors
        except:
            currentNeighbors = "0"
            neighbors = ['0']
        else:
            if currentNeighbors == "r":      #reset
                currentNeighbors = "a"
                neighbors = ['0']
            else:
                currentNeighbors = "r" 
                neighbors = ['a']

        if clusterCount == 1:
            currentNeighbors = "r"

        main.log.info("Cluster Count = " + str(clusterCount))

        intentsRate = main.params['METRICS']['intents_rate']
        intentsWithdrawn = main.params[ 'METRICS' ][ 'intents_withdrawn' ]
        intentsFailed  = main.params[ 'METRICS' ][ 'intents_failed' ]
        testDuration = main.params[ 'TEST' ][ 'duration' ]
        logInterval = main.params[ 'TEST' ][ 'log_interval' ]
        debug = main.params[ 'debugMode' ]
        numKeys = main.params[ 'TEST' ][ 'numKeys' ]
        cyclePeriod = main.params[ 'TEST' ][ 'cyclePeriod' ]
        #neighbors = (main.params[ 'TEST' ][ 'neighbors' ]).split(",") 
        metricList = [intentsRate, intentsWithdrawn, intentsFailed]

        for n in range(0, len(neighbors)): 
            if neighbors[n] == 'a': 
                neighbors[n] = str(clusterCount -1)
                if int(clusterCount) == 1:
                    neighbors = neighbors.pop()
 
        for n in neighbors:
            main.log.info("Run with " + n + " neighbors") 
            time.sleep(5)
            main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numKeys " + numKeys )
            main.ONOSbench.handle.expect(":~")
            main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller numNeighbors " + n ) 
            main.ONOSbench.handle.expect(":~")
            main.ONOSbench.handle.sendline("onos $OC1 cfg set org.onosproject.intentperf.IntentPerfInstaller cyclePeriod " + cyclePeriod )
            main.ONOSbench.handle.expect(":~")

            cmd = "onos $OC1 intent-perf-start"
            main.ONOSbench.handle.sendline(cmd)
            main.ONOSbench.handle.expect(":~")
            main.log.info("Starting ONOS (all nodes)  intent-perf from $OC1" )

            main.log.info( "Starting test loop for " + str(testDuration) + " seconds...\n" )
            stop = time.time() + float( testDuration )

            while time.time() < stop:
                time.sleep( float( logInterval ) )
                groupResult = []
                for node in range (1, clusterCount + 1):
                    groupResult.append(0)

                    cmd = " onos-ssh $OC" + str(node) +  """ cat /opt/onos/log/karaf.log | grep "SNAPSHOT | Throughput" | tail -1  """
                    main.log.info("COMMAND: " + str(cmd))

                    x = 0
                    while True:
                        main.ONOSbench.handle.sendline(cmd)
                        main.ONOSbench.handle.expect(":~")
                        raw = main.ONOSbench.handle.before
                        if "OVERALL=" in raw:
                            break
                        x += 1
                        if x > 10:
                            main.log.error("Expected output not being recieved... continuing")
                            break
                        time.sleep(2)

                    raw = raw.splitlines()
                    splitResults = []
                    for line in raw:
                        splitResults.extend(line.split(" "))

                    myResult = "--"
                    for field in splitResults:
                        if "OVERALL" in field:
                            myResult = field

                    if myResult == "--":
                        main.log.error("Parsing/Pexpect error\n" + str(splitResults))

                    myResult = myResult.replace(";", "")
                    myResult = myResult.replace("OVERALL=","")
                    myResult = float(myResult)
                    groupResult[len(groupResult) -1] = myResult

                    main.log.info("Node " + str(node) + " overall rate: " + str(myResult))

                clusterTotal = str(numpy.sum(groupResult))
                main.log.report("Results from this round of polling: " + str(groupResult))
                main.log.report("Cluster Total: " + clusterTotal + "\n")

            cmd = "onos $OC1 intent-perf-stop"
            main.ONOSbench.handle.sendline(cmd)
            main.ONOSbench.handle.expect(":~")
            main.log.info("Stopping intentperf" )
   
            resultsDB = open("IntentEventTPDB", "a")
            for node in groupResult: 

                resultString = "'" + commit + "',"
                resultString += "'1gig',"
                resultString += str(clusterCount) + ","
                resultString += "'baremetal" + str(int(groupResult.index(node)) + 1) + "',"
                resultString += n + ","
                resultString += str(node) + ","
                resultString += str(0) + "\n" #no stddev
                resultsDB.write(resultString)
            
            resultsDB.close() 
            
            main.ONOSbench.onosErrorLog(ONOSIp[1])