TestON/tests/SCPF/SCPFflowTp1g/SCPFflowTp1g.py

"""
Copyright 2015 Open Networking Foundation ( ONF )

Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>

    TestON is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 2 of the License, or
    ( at your option ) any later version.

    TestON is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with TestON.  If not, see <http://www.gnu.org/licenses/>.
"""
# ScaleOutTemplate -> flowTP
#
# CASE1 starts number of nodes specified in param file
#
# cameron@onlab.us

import sys
import os.path


class SCPFflowTp1g:

    def __init__( self ):
        self.default = ''

    def CASE0( self, main ):
        import time
        try:
            from tests.dependencies.ONOSSetup import ONOSSetup
            main.testSetUp = ONOSSetup()
        except ImportError:
            main.log.error( "ONOSSetup not found. exiting the test" )
            main.cleanAndExit()
        main.testSetUp.envSetupDescription()
        try:
            # Load values from params file
            cellName = main.params[ 'ENV' ][ 'cellName' ]
            main.apps = main.params[ 'ENV' ][ 'cellApps' ]
            BENCHUser = main.params[ 'BENCH' ][ 'user' ]
            BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
            main.scale = ( main.params[ 'SCALE' ] ).split( "," )
            main.flowRuleCfg = main.params[ 'CFG' ][ 'flowRule' ]
            main.neighbor = ( main.params[ 'TEST' ][ 'neighbors' ] ).split( "," )
            main.nullProviderCfg = main.params[ 'CFG' ][ 'nullProvider' ]
            isFlowObj = main.params[ 'TEST' ][ 'flowObj' ] == "True"
            if isFlowObj:
                resultFile = main.params[ 'TEST' ][ 'flowObjResultFile' ]
            else:
                resultFile = main.params[ 'TEST' ][ 'flowResultFile' ]
            stepResult = main.testSetUp.envSetup()
            resultsDB = open( str( resultFile ), "w+" )
            resultsDB.close()
        except Exception as e:
            main.testSetUp.envSetupException( e )
        main.testSetUp.evnSetupConclusion( stepResult )
        main.commit = ( main.commit.split( " " ) )[ 1 ]

    def CASE1( self, main ):
        main.testSetUp.ONOSSetUp( main.Cluster, True, cellName=cellName )
        main.log.info( "Startup sequence complete" )
        main.ONOSbench.logReport( main.Cluster.active( 0 ).ipAddress, [ "ERROR", "WARNING", "EXCEPT" ], outputMode="d" )

    def CASE2( self, main ):
        #
        # This is the flow/flowObjective TP test
        #
        import os.path
        import numpy
        import math
        import time
        import datetime
        import traceback

        testCMD = [ 0, 0, 0, 0 ]
        warmUp = int( main.params[ 'TEST' ][ 'warmUp' ] )
        sampleSize = int( main.params[ 'TEST' ][ 'sampleSize' ] )
        switches = int( main.params[ 'TEST' ][ 'switches' ] )
        testCMD[ 0 ] = main.params[ 'TEST' ][ 'testCMD0' ]
        testCMD[ 1 ] = main.params[ 'TEST' ][ 'testCMD1' ]
        testCMD[ 2 ] = main.params[ 'TEST' ][ 'testCMD2' ]
        testCMD[ 3 ] = main.params[ 'TEST' ][ 'testCMD3' ]
        flowObjType = main.params[ 'TEST' ][ 'flowObjType' ]
        cooldown = main.params[ 'TEST' ][ 'cooldown' ]
        cellName = main.params[ 'ENV' ][ 'cellName' ]
        BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
        BENCHUser = main.params[ 'BENCH' ][ 'user' ]
        MN1Ip = main.params[ 'MN' ][ 'ip1' ]
        homeDir = os.path.expanduser( '~' )
        flowRuleBackup = str( main.params[ 'TEST' ][ 'enableFlowRuleStoreBackup' ] )
        main.log.info( "Flow Rule Backup is set to:" + flowRuleBackup )

        servers = str( main.Cluster.numCtrls )
        neighbors = '0' if main.neighbor.pop( 0 ) == '0' else str( main.Cluster.numCtrls - 1 )
        main.log.info( "Number of Neighbors: " + neighbors )
        ts = time.time()
        st = datetime.datetime.fromtimestamp( ts ).strftime( '%Y-%m-%d %H:%M:%S' )
        main.step( "\tSTARTING TEST" )
        main.step( "\tLOADING FROM SERVERS:  \t" + str( main.Cluster.numCtrls ) )
        main.step( "\tNEIGHBORS:\t" + neighbors )
        main.log.info( "=============================================================" )
        main.log.info( "=============================================================" )
        # write file to configure nil link
        ipCSV = ""
        for i in range( main.Cluster.maxCtrls ):
            tempstr = "ip" + str( i + 1 )
            ipCSV += main.params[ 'CTRL' ][ tempstr ]
            if i + 1 < main.Cluster.maxCtrls:
                ipCSV += ","

        main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
                                   main.flowRuleCfg,
                                   "backupCount 1" )
        for i in range( 3 ):
            main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
                                       main.nullProviderCfg,
                                       "deviceCount 35" )
            main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
                                       main.nullProviderCfg,
                                       "topoShape linear" )
            main.ONOSbench.onosCfgSet( main.Cluster.active( 0 ).ipAddress,
                                       main.nullProviderCfg,
                                       "enabled true" )

            time.sleep( 5 )
            main.ONOSbench.handle.sendline( "onos $OC1 summary" )
            main.ONOSbench.handle.expect( ":~" )
            check = main.ONOSbench.handle.before
            main.log.info( "\nStart up check: \n" + check + "\n" )
            if "SCC(s)=1," in check:
                time.sleep( 5 )
                main.ONOSbench.handle.sendline( "onos $OC1 balance-masters" )
                main.ONOSbench.handle.expect( ":~" )
                time.sleep( 5 )
                main.ONOSbench.handle.sendline( "onos $OC1 roles " )
                main.ONOSbench.handle.expect( ":~" )
                main.log.info( "switch masterships:" + str( main.ONOSbench.handle.before ) )
                break
            time.sleep( 5 )

        # divide flows/flowObjectives
        if isFlowObj:
            toInstall = "FlowObjectives"
            installCount = int( main.params[ 'TEST' ][ 'flowObjectives' ] )
            ifFailed = "FLOW_OBJ_TESTER.PY FAILURE"
            resultFile = main.params[ 'TEST' ][ 'flowObjResultFile' ]
        else:
            toInstall = "Flows"
            installCount = int( main.params[ 'TEST' ][ 'flows' ] )
            ifFailed = "FLOW_TESTER.PY FAILURE"
            resultFile = main.params[ 'TEST' ][ 'flowResultFile' ]
        main.log.info( toInstall + " Target  = " + str( installCount ) )

        installCountPerSwitch = ( installCount * max( int( neighbors ) + 1, int( servers ) ) ) / ( ( int( neighbors ) + 1 ) * int( servers ) * ( switches ) )

        main.log.info( toInstall + " per switch = " + str( installCountPerSwitch ) )
        # build list of servers in "$OC1, $OC2...." format
        serverEnvVars = ""
        for i in range( int( servers ) ):
            serverEnvVars += ( "-s " + main.Cluster.active( i ).ipAddress + " " )

        data = [ [ "" ] * int( servers ) ] * int( sampleSize )
        maxes = [ "" ] * int( sampleSize )

        flowCMD = "python3 " + homeDir + "/onos/tools/test/bin/"
        if isFlowObj:
            flowCMD += testCMD[ 2 ] + " " + str( installCountPerSwitch ) + " " + testCMD[ 1 ]
            flowCMD += " " + neighbors + " " + testCMD[ 3 ] + " " + str( flowObjType ) + " " + str( serverEnvVars ) + "-j"
        else:
            flowCMD += testCMD[ 0 ] + " " + str( installCountPerSwitch ) + " " + testCMD[ 1 ]
            flowCMD += " " + neighbors + " " + str( serverEnvVars ) + "-j"

        main.log.info( flowCMD )
        # time.sleep( 60 )

        for test in range( 0, warmUp + sampleSize ):
            if test < warmUp:
                main.log.info( "Warm up " + str( test + 1 ) + " of " + str( warmUp ) )
            else:
                main.log.info( "====== Test run: " + str( test - warmUp + 1 ) + " ======" )

            main.ONOSbench.handle.sendline( flowCMD )
            main.ONOSbench.handle.expect( ":~" )
            rawResult = main.ONOSbench.handle.before
            main.log.info( "Raw results: \n" + rawResult + "\n" )

            if "failed" in rawResult:
                main.log.report( ifFailed )
                main.log.report( " \n" + rawResult + " \n" )
                for ctrl in main.Cluster.active():
                    main.log.report( "=======================================================" )
                    main.log.report( ctrl.name + "LOG REPORT" )
                    main.ONOSbench.logReport( ctrl.ipAddress, [ "ERROR", "WARNING", "EXCEPT" ], outputMode="d" )
                main.ONOSbench.handle.sendline( "onos $OC1 flows" )
                main.ONOSbench.handle.expect( ":~" )
                main.log.info( main.ONOSbench.handle.before )

                break
            result = [ "" ] * ( main.Cluster.numCtrls )

            rawResult = rawResult.splitlines()

            for node in range( main.Cluster.numCtrls ):
                for line in rawResult:
                    # print( "line: " + line )
                    if main.Cluster.active( node ).ipAddress in line and "server" in line:
                        temp = line.split( " " )
                        for word in temp:
                            # print ( "word: " + word )
                            if "elapsed" in repr( word ):
                                index = temp.index( word ) + 1
                                myParsed = ( temp[ index ] ).replace( ",", "" )
                                myParsed = myParsed.replace( "}", "" )
                                myParsed = int( myParsed )
                                result[ node ] = myParsed
                                main.log.info( main.Cluster.active( node ).ipAddress + " : " + str( myParsed ) )
                                break

            if test >= warmUp:
                for i in result:
                    if i == "":
                        main.log.error( "Missing data point, critical failure incoming" )

                print result
                maxes[ test - warmUp ] = max( result )
                main.log.info( "Data collection iteration: " + str( test - warmUp ) + " of " + str( sampleSize ) )
                main.log.info( "Throughput time: " + str( maxes[ test - warmUp ] ) + "(ms)" )

                data[ test - warmUp ] = result

            # wait for flows = 0
            for checkCount in range( 0, 5 ):
                time.sleep( 10 )
                main.ONOSbench.handle.sendline( "onos $OC1 summary" )
                main.ONOSbench.handle.expect( ":~" )
                flowCheck = main.ONOSbench.handle.before
                if "flows=0," in flowCheck:
                    main.log.info( toInstall + " removed" )
                    break
                else:
                    for line in flowCheck.splitlines():
                        if "flows=" in line:
                            main.log.info( "Current Summary: " + line )
                if checkCount == 2:
                    main.log.info( toInstall + " are stuck, moving on " )
            time.sleep( 5 )

        main.log.info( "raw data: " + str( data ) )
        main.log.info( "maxes:" + str( maxes ) )

        # report data
        print( "" )
        main.log.info( "\t Results (measurments are in milliseconds)" )
        print( "" )

        nodeString = ""
        for i in range( 1, int( servers ) + 1 ):
            nodeString += ( "\tNode " + str( i ) )

        for test in range( 0, sampleSize ):
            main.log.info( "\t Test iteration " + str( test + 1 ) )
            main.log.info( "\t------------------" )
            main.log.info( nodeString )
            resultString = ""

            for i in range( 0, int( servers ) ):
                resultString += ( "\t" + str( data[ test ][ i ] ) )
            main.log.info( resultString )
            print( "\n" )

        try:
            avgOfMaxes = numpy.mean( maxes )
            main.log.info( "Average of max value from each test iteration: " + str( avgOfMaxes ) )
            stdOfMaxes = numpy.std( maxes )
            main.log.info( "Standard Deviation of max values: " + str( stdOfMaxes ) )
            print( "\n\n" )
        except TypeError:
            main.cleanAndExit( msg="flow-tester execution failed" )

        avgTP = int( installCount ) / avgOfMaxes  # result in kflows/second

        tp = []
        for i in maxes:
            tp.append( ( int( installCount ) / i ) )

        stdTP = numpy.std( tp )

        main.log.info( "Average thoughput:  " + str( avgTP ) + " K " + toInstall + "/second" )
        main.log.info( "Standard deviation of throughput: " + str( stdTP ) + " K " + toInstall + "/second" )

        # Check if throughput result is abnormal
        if isFlowObj:
            threshold = float( main.params[ 'ALARM' ][ 'minTpFlowObj' ].split( ',' )[ main.cycle - 1 ] )
        else:
            threshold = float( main.params[ 'ALARM' ][ 'minTp' ].split( ',' )[ main.cycle - 1 ] )
        if avgTP < threshold:
            main.log.alarm( "{}-node with {} neighbor: {}K/s < {}K/s".format( main.Cluster.numCtrls,
                                                                              neighbors, avgTP, threshold ) )

        resultsLog = open( str( resultFile ), "a" )
        resultString = ( "'" + main.commit + "'," )
        resultString += ( "'1gig'," )
        resultString += ( str( installCount ) + "," )
        resultString += ( str( main.Cluster.numCtrls ) + "," )
        resultString += ( neighbors + "," )
        resultString += ( str( avgTP ) + "," + str( stdTP ) + "\n" )
        resultsLog.write( resultString )
        resultsLog.close()

        main.log.report( "Result line to file: " + resultString )

        main.ONOSbench.logReport( main.Cluster.active( 0 ).ipAddress, [ "ERROR", "WARNING", "EXCEPT" ], outputMode="d" )