TestON/tests/IntentEventTP/IntentEventTP.py

# ScaleOutTemplate --> IntentEventTP
#
# CASE1 starts number of nodes specified in param file
#
# cameron@onlab.us

import sys
import os
import time


class IntentEventTP:

    def __init__( self ):
        self.default = ''

    def CASE1( self, main ):            #This is the initialization case
        import os.path                  #this case will clean up all nodes 
        import time                     #but only node 1 is started in this case
        
        global clusterCount             #number of nodes running
        global ONOSIp                   #list of ONOS IP addresses 
        clusterCount = 1
        ONOSIp = [ 0 ]

        #Load values from params file
        checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
        gitPull = main.params[ 'GIT' ][ 'autopull' ]
        cellName = main.params[ 'ENV' ][ 'cellName' ]
        Features= main.params[ 'ENV' ][ 'cellFeatures' ]
        BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
        BENCHUser = main.params[ 'BENCH' ][ 'user' ]
        maxNodes = int(main.params[ 'availableNodes' ])
        MNip = main.params[ 'MN' ][ 'ip1' ]
        skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
        numSwitches = main.params[ 'TEST' ][ 'numSwitches' ]

        homeDir = os.path.expanduser('~')
        
        main.ONOSbench.handle.sendline("export TERM=vt100")
        dump = main.ONOSbench.handle.expect(":~")

        #Populate ONOSIp with ips from params 
        for i in range(1, maxNodes + 1): 
            ipString = 'ip' + str(i) 
            ONOSIp.append(main.params[ 'CTRL' ][ ipString ])   
        
        #kill off all onos processes
        main.log.step("Safety check, killing all ONOS processes")
        main.log.step("before initiating enviornment setup")
        for node in range(1, maxNodes + 1):
            main.log.info("killing node " + str(node))
            main.ONOSbench.onosDie(ONOSIp[node])

        #construct the cell file
        main.log.info("Creating cell file")
        exec "a = main.ONOSbench.createCellFile"
        cellIp = []
        for node in range (1, clusterCount + 1):
            cellIp.append(ONOSIp[node])
        a(BENCHIp,cellName,MNip,str(Features), *cellIp)   

        main.step( "Set Cell" )
        main.ONOSbench.setCell(cellName)

        #Uninstall everywhere
        main.log.step( "Cleaning Enviornment..." )
        for i in range(1, maxNodes + 1):
            main.log.info(" Uninstalling ONOS " + str(i) )
            main.ONOSbench.onosUninstall( ONOSIp[i] )

        #git 
        main.step( "Git checkout and pull " + checkoutBranch )
        if gitPull == 'on':
            checkoutResult = main.ONOSbench.gitCheckout( checkoutBranch )
            pullResult = main.ONOSbench.gitPull()
        else:
            checkoutResult = main.TRUE
            pullResult = main.TRUE
            main.log.info( "Skipped git checkout and pull" )
        
        #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
        if skipMvn != "yes":
            mvnResult = main.ONOSbench.cleanInstall()
 
        #configure null device provider         
        switchList = [0,int(numSwitches),0,0,0,0,0,0]
        devicesString  = ""
        for node in range(1, maxNodes + 1):
            devicesString += (ONOSIp[node] + ":" + str(switchList[node] ))
            if node < maxNodes:
                devicesString += (",")

        main.log.info("Configuring device provider: ONOS 1 with " + (numSwitches) + " switches")
        localPath = "/onos/tools/package/etc/org.onosproject.provider.nil.device.impl.NullDeviceProvider.cfg"
        filePath = homeDir + localPath
        main.log.info(filePath)

        configFile = open(filePath, 'w+')
        configFile.write("devConfigs = " + devicesString + "\n")
        configFile.write("#numPorts = 8") 
        configFile.close()
        main.log.info("DevConfig = " + devicesString) 
        main.log.info("Device provider file written and closed")

        ## configuring null link provider
        main.log.info(" Configuring null provider to disable flicker" )
        homeDir = os.path.expanduser('~')
        main.log.info(homeDir)
        localPath = "/onos/tools/package/etc/org.onosproject.provider.nil.link.impl.NullLinkProvider.cfg"
        filePath = homeDir + localPath
        main.log.info(filePath)

        neighborsString = ""
        for node in range(1, maxNodes + 1):
            neighborsString += ONOSIp[node]
            if node < maxNodes:
                neighborsString += ","

        configFile = open(filePath, 'w+')
        configFile.write("#eventRate =\n")
        configFile.write("#cfgFile = /tmp/foo.cfg        #If enabled, points to the full path to the topology file.\n")
        configFile.write("#neighbors = ")
        configFile.close()
        main.log.info("Configuration completed")
        
        main.log.info("Writing link graph configuration file..." )
        homeDir = os.path.expanduser('~')
        localPath = "/onos/tools/package/etc/linkGraph.cfg"
        filePath = homeDir + localPath
        linkGraph = open(filePath, 'w+')
        linkGraph.write("# NullLinkProvider topology description (config file).\n")
        linkGraph.write("# The NodeId is only added if the destination is another node's device.\n")
        linkGraph.write("# Bugs: Comments cannot be appended to a line to be read.\n")

        myPort = 6
        for node in range(1, clusterCount+1):
            linkGraph.write("graph " + ONOSIp[node] + " {\n")
            for switch in range (0, switchList[node]-1):
                line = ""
                line = ("\t" + str(switch) + ":" + str(myPort))
                line += " -- "
                line += (str(switch+1) + ":" + str(myPort-1) + "\n")
                linkGraph.write(line) 
            linkGraph.write("}")
        linkGraph.close()        

        main.step( "Creating ONOS package" )
        packageResult = main.ONOSbench.onosPackage()  

        main.step( "Installing ONOS package" )
        install1Result = main.ONOSbench.onosInstall( node=ONOSIp[1] )

        main.step( "verify cells" )
        verifyCellResult = main.ONOSbench.verifyCell()

        main.step( "Checking if ONOS is up yet" )
        for i in range( 2 ):
            isup = main.ONOSbench.isup( ONOSIp[1] )
            if isup:
                break
        if not isup:
            main.log.report( "ONOS1 didn't start!" )

        lastOutput = "--" 
        origin = time.time()
        clockStarted = False 
        while True:
            main.ONOSbench.handle.sendline("onos $OC1 summary")
            main.ONOSbench.handle.expect(":~")
            clusterCheck = ((main.ONOSbench.handle.before).splitlines())[3]
            print("\nBefore: " + str(clusterCheck))
            if "SCC(s)=1," in clusterCheck and ("devices=" + str(numSwitches)) in clusterCheck:                  #check for links and devices too 
                break 
            if clusterCheck != lastOutput:
                sameOutput = False 
            elif clusterCheck == lastOutput:
                if clockStarted == False: 
                    start = time.time()
                    clockStarted = True
                if time.time() > (start + 30):
                    main.log.error("TIMEOUT EXCEEDED: Clusters have not converged, continuing anyway...") 
                    break 
            lastOutput = clusterCheck
            time.sleep(5)



    def CASE2( self, main ):
        # This case increases the cluster size by whatever scale is
        # Note: 'scale' is the size of the step
        # if scaling is not a part of your test, simply run this case
        # once after CASE1 to set up your enviornment for your desired 
        # cluster size. If scaling is a part of you test call this case each time 
        # you want to increase cluster size

        ''                                                         
        'Increase number of nodes and initiate CLI'
        ''
        import time
        import os.path
        global clusterCount
        
        Features= main.params[ 'ENV' ][ 'cellFeatures' ] 
        cellName = main.params[ 'ENV' ][ 'cellName' ]
        MNip = main.params[ 'MN' ][ 'ip1' ]
        BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
        numSwitches = int(main.params[ 'TEST' ][ 'numSwitches' ])
        scale = int( main.params[ 'SCALE' ] )
        maxNodes = int(main.params[ 'availableNodes' ])
        clusterCount += scale
        homeDir = os.path.expanduser('~')        

        #kill off all onos processes
        main.log.step("Safety check, killing all ONOS processes")
        main.log.step("before initiating enviornment setup")
        for node in range(1, maxNodes + 1):
            main.ONOSbench.onosDie(ONOSIp[node]) 

        #Uninstall everywhere
        main.log.step( "Cleaning Enviornment..." )
        for i in range(1, maxNodes + 1):
            main.log.info(" Uninstalling ONOS " + str(i) )
            main.ONOSbench.onosUninstall( ONOSIp[i] )      
        
        #construct the cell file
        main.log.info("Creating cell file")
        exec "a = main.ONOSbench.createCellFile"
        cellIp = []
        for node in range (1, clusterCount + 1):
            cellIp.append(ONOSIp[node])
        a(BENCHIp,cellName,MNip,str(Features), *cellIp)

        main.step( "Set Cell" )
        main.ONOSbench.setCell(cellName)

        baselineSwitchCount = numSwitches/clusterCount
        switchList = [0,0,0,0,0,0,0,0]
        
        for node in range(1, clusterCount + 1):
            switchList[node] = baselineSwitchCount  
            
        for node in range(1, (numSwitches%clusterCount)+1):
            switchList[node] += 1
                      
        devicesString  = ""
        for node in range(1, maxNodes + 1):
            devicesString += (ONOSIp[node] + ":" + str(switchList[node] ))
            if node < maxNodes:
                devicesString += (",")

        main.log.info("Configuring device provider")
        localPath = "/onos/tools/package/etc/org.onosproject.provider.nil.device.impl.NullDeviceProvider.cfg"
        filePath = homeDir + localPath
        main.log.info(filePath)

        configFile = open(filePath, 'w+')
        configFile.write("devConfigs = " + devicesString +"\n")
        configFile.write("# numPorts = 8")
        configFile.close()
        main.log.info("DevConfig = " + devicesString)
        main.log.info("Device provider file written and closed")

        main.log.info("Writing link graph configuration file..." )
        homeDir = os.path.expanduser('~')
        localPath = "/onos/tools/package/etc/linkGraph.cfg"
        filePath = homeDir + localPath
        linkGraph = open(filePath, 'w+')
        linkGraph.write("# NullLinkProvider topology description (config file).\n")
        linkGraph.write("# The NodeId is only added if the destination is another node's device.\n")
        linkGraph.write("# Bugs: Comments cannot be appended to a line to be read.\n")

        myPort = 6
        for node in range(1, clusterCount+1):
            linkGraph.write("graph " + ONOSIp[node] + " {\n")
            for switch in range (0, switchList[node]-1):
                line = ""
                line = ("\t" + str(switch) + ":" + str(myPort))
                line += " -- "
                line += (str(switch+1) + ":" + str(myPort-1) + "\n")
                linkGraph.write(line)
            linkGraph.write("}\n")
        linkGraph.close()

        main.step( "Creating ONOS package, preparing to reinstall" )
        packageResult = main.ONOSbench.onosPackage()   
       
        main.log.report( "Reinstalling on all nodes and increasing cluster size to " + str( clusterCount ) )
        for node in range(1, clusterCount + 1):
            main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])    
            main.ONOSbench.onosInstall( ONOSIp[node])
    
            for i in range( 2 ):
                isup = main.ONOSbench.isup( ONOSIp[node] )
                if isup:
                    main.log.info("ONOS " + str(node) + " is up\n")
                    break
            if not isup:
                main.log.report( "ONOS " + str(node) + " didn't start!" )
        
        lastOutput = "--"
        origin = time.time()
        clockStarted = False
        while True:
            main.ONOSbench.handle.sendline("onos $OC1 summary")
            main.ONOSbench.handle.expect(":~")
            clusterCheck = ((main.ONOSbench.handle.before).splitlines())[3]
            print("\nBefore: " + str(clusterCheck))
            if "SCC(s)=1," in clusterCheck and ("nodes=" + str(clusterCount)) in clusterCheck and ("devices=" + str(numSwitches)) in clusterCheck:  
                break
            if clusterCheck != lastOutput:
                sameOutput = False
            elif clusterCheck == lastOutput:
                if clockStarted == False:
                    start = time.time()
                    clockStarted = True
                if time.time() > (start + 60):
                    main.log.error("TIMEOUT EXCEEDED: Clusters have not converged, continuing anyway...")
                    break
            lastOutput = clusterCheck
            time.sleep(5)
        
            
    def CASE3( self, main ):   
        import time
        import json
        import string 
        import csv

        main.log.info("Cluster Count = " + str(clusterCount))

        intentsRate = main.params['METRICS']['intents_rate']
        intentsWithdrawn = main.params[ 'METRICS' ][ 'intents_withdrawn' ]
        intentsFailed  = main.params[ 'METRICS' ][ 'intents_failed' ]
        testDuration = main.params[ 'TEST' ][ 'duration' ]
        logInterval = main.params[ 'TEST' ][ 'log_interval' ]
        debug = main.params[ 'debugMode' ]

        metricList = [intentsRate, intentsWithdrawn, intentsFailed]
        
        tempsleep =40
        main.log.info("sleeping " + str(tempsleep)) 
        time.sleep(tempsleep)
        
        loadFrom = ['0']
        loadFrom.extend((main.params[ 'TEST' ][ 'loadFrom' ]).split(","))

        for node in range(1, clusterCount+1):
            if loadFrom[node] == "1": 
                cmd = "onos $OC" + str(node) + " feature:install onos-app-intent-perf"
                main.ONOSbench.handle.sendline(cmd)
                main.ONOSbench.handle.expect(":~")
                main.log.info("Load initiated on node " + str(node))
       
        main.log.info( "Starting test loop for " + str(testDuration) + " seconds...\n" )
        stop = time.time() + float( testDuration )
        
        while time.time() < stop:
            time.sleep( float( logInterval ) )
            for node in range (1, clusterCount + 1):
                myResults = ['0','0','0']
                for metric in metricList: 

                    onosEnv = "onos $OC" + str(node)  
                    cmd = onosEnv + " " + metric
                    main.log.info("COMMAND: " + cmd)
                    main.ONOSbench.handle.sendline( cmd )
                    time.sleep(10)
                    main.ONOSbench.handle.expect(":~")                   
                    rawResult = main.ONOSbench.handle.before
                    rawResult = (rawResult.splitlines())

                    tempResult = "--"
                    for word in rawResult:
                        if debug: print("word: " + word)
                        if "m1" in str(word): 
                            tempResult = word
                            break

                    if tempResult == "--": 
                        main.log.error("WRONG pexepct.before data\n" + str(rawResult))
                        main.log.info("retrying command... ")
                        main.ONOSbench.handle.sendline(cmd)
                        main.ONOSbench.handle.expect(":~")
                        test = main.ONOSbench.handle.before
                        print ("\n\n" + str(test))

                    tempResult = round(float(tempResult.replace("m1=","")),1)
                    tempResult = str(tempResult)                        # easy way to clean up number/prep to log
                    resultIndex = metricList.index(metric)
                    myResults[resultIndex] = tempResult
                
                main.log.info("\tNode " + str(node))
                main.log.info("Installed\tWithdrawn\tFailed")
                main.log.info(myResults[0] + "\t\t " + myResults[1] + "\t\t" + myResults[2] + "\n")