TestON/tests/dependencies/topology.py

"""
Copyright 2016 Open Networking Foundation ( ONF )

Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>

    TestON is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 2 of the License, or
    ( at your option ) any later version.

    TestON is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with TestON.  If not, see <http://www.gnu.org/licenses/>.
"""
import time
import re
import imp
import json
from core import utilities


class Topology:

    def __init__( self ):
        self.default = ''

    """
        These functions can be used for topology comparisons
    """
    def getAll( self, function, needRetry=False, kwargs={}, inJson=False ):
        """
        Description:
            get all devices/links/hosts/ports of the onosCli
        Required:
            * function - name of the function
            * needRetry - it will retry if this is true.
            * kwargs - kwargs of the function
            * inJson - True if want it in Json form
        Returns:
            Returns the list of the result.
        """
        returnList = []
        threads = []
        for ctrl in main.Cluster.active():
            func = getattr( ctrl.CLI, function )
            t = main.Thread( target=utilities.retry if needRetry else func,
                             name=function + "-" + str( ctrl ),
                             args=[ func, [ None ] ] if needRetry else [],
                             kwargs=kwargs )
            threads.append( t )
            t.start()

        for t in threads:
            t.join()
            if inJson:
                try:
                    returnList.append( json.loads( t.result ) )
                except ( ValueError, TypeError ):
                    main.log.exception( "Error parsing hosts results" )
                    main.log.error( repr( t.result ) )
                    returnList.append( None )
            else:
                returnList.append( t.result )
        return returnList

    def compareDevicePort( self, Mininet, controller, mnSwitches, devices, ports ):
        """
        Description:
            compares the devices and port of the onos to the mininet.
        Required:
            * Mininet - mininet driver to use
            * controller - controller position of the devices
            * mnSwitches - switches of mininet
            * devices - devices of the onos
            * ports - ports of the onos
        Returns:
            Returns main.TRUE if the results are matching else
            Returns main.FALSE
        """
        if devices[ controller ] and ports[ controller ] and \
                        "Error" not in devices[ controller ] and \
                        "Error" not in ports[ controller ]:
            try:
                currentDevicesResult = Mininet.compareSwitches(
                    mnSwitches,
                    json.loads( devices[ controller ] ),
                    json.loads( ports[ controller ] ) )
            except ( TypeError, ValueError ):
                main.log.error(
                    "Could not load json: {0} or {1}".format( str( devices[ controller ] ),
                                                              str( ports[ controller ] ) ) )
                currentDevicesResult = main.FALSE
        else:
            currentDevicesResult = main.FALSE
        return currentDevicesResult

    def compareBase( self, compareElem, controller, compareF, compareArg ):
        """
        Description:
            compares the links/hosts of the onos to the mininet.
        Required:
            * compareElem - list of links/hosts of the onos
            * controller - controller position of the devices
            * compareF - function of the mininet that will compare the
            results
            * compareArg - arg of the compareF.
        Returns:
            Returns main.TRUE if the results are matching else
            Returns main.FALSE
        """
        if compareElem[ controller ] and "Error" not in compareElem[ controller ]:
            try:
                if isinstance( compareArg, list ):
                    compareArg.append( json.loads( compareElem[ controller ] ) )
                else:
                    compareArg = [ compareArg, json.loads( compareElem[ controller ] ) ]

                currentCompareResult = compareF( *compareArg )
            except( TypeError, ValueError ):
                main.log.error(
                    "Could not load json: {0} or {1}".format( str( compareElem[ controller ] ) ) )
                currentCompareResult = main.FALSE
        else:
            currentCompareResult = main.FALSE

        return currentCompareResult

    def compareTopos( self, Mininet, attempts=1, includeCaseDesc=True ):
        """
        Description:
            compares the links and hosts and switches of the onos to the mininet.
        Required:
            * Mininet - Mininet driver to use.
            * attempts - number of attempts to compare in case
            the result is different after a certain time.
        Returns:
            Returns main.TRUE if the results are matching else
            Returns main.FALSE
        """
        if includeCaseDesc:
            main.case( "Compare ONOS Topology view to Mininet topology" )
            main.caseExplanation = "Compare topology elements between Mininet" +\
                                   " and ONOS"
        main.log.info( "Gathering topology information from Mininet" )
        devicesResults = main.FALSE  # Overall Boolean for device correctness
        linksResults = main.FALSE  # Overall Boolean for link correctness
        hostsResults = main.FALSE  # Overall Boolean for host correctness
        deviceFails = []  # Nodes where devices are incorrect
        linkFails = []  # Nodes where links are incorrect
        hostFails = []  # Nodes where hosts are incorrect

        mnSwitches = Mininet.getSwitches()
        mnLinks = Mininet.getLinks()
        mnHosts = Mininet.getHosts()

        main.step( "Comparing Mininet topology to ONOS topology" )

        while ( attempts >= 0 ) and\
                ( not devicesResults or not linksResults or not hostsResults ):
            main.log.info( "Sleeping {} seconds".format( 2 ) )
            time.sleep( 2 )
            if not devicesResults:
                devices = self.getAll( "devices", False )
                ports = self.getAll( "ports", False )
                devicesResults = main.TRUE
                deviceFails = []  # Reset for each failed attempt
            if not linksResults:
                links = self.getAll( "links", False )
                linksResults = main.TRUE
                linkFails = []  # Reset for each failed attempt
            if not hostsResults:
                hosts = self.getAll( "hosts", False )
                hostsResults = main.TRUE
                hostFails = []  # Reset for each failed attempt

            #  Check for matching topology on each node
            for controller in main.Cluster.getRunningPos():
                controllerStr = str( controller + 1 )  # ONOS node number
                # Compare Devices
                currentDevicesResult = self.compareDevicePort( Mininet, controller,
                                                               mnSwitches,
                                                               devices, ports )
                if not currentDevicesResult:
                    deviceFails.append( controllerStr )
                devicesResults = devicesResults and currentDevicesResult
                # Compare Links
                currentLinksResult = self.compareBase( links, controller,
                                                       Mininet.compareLinks,
                                                       [ mnSwitches, mnLinks ] )
                if not currentLinksResult:
                    linkFails.append( controllerStr )
                linksResults = linksResults and currentLinksResult
                # Compare Hosts
                currentHostsResult = self.compareBase( hosts, controller,
                                                           Mininet.compareHosts,
                                                           mnHosts )
                if not currentHostsResult:
                    hostFails.append( controllerStr )
                hostsResults = hostsResults and currentHostsResult
            # Decrement Attempts Remaining
            attempts -= 1

        utilities.assert_equals( expect=[],
                                 actual=deviceFails,
                                 onpass="ONOS correctly discovered all devices",
                                 onfail="ONOS incorrectly discovered devices on nodes: " +
                                 str( deviceFails ) )
        utilities.assert_equals( expect=[],
                                 actual=linkFails,
                                 onpass="ONOS correctly discovered all links",
                                 onfail="ONOS incorrectly discovered links on nodes: " +
                                 str( linkFails ) )
        utilities.assert_equals( expect=[],
                                 actual=hostFails,
                                 onpass="ONOS correctly discovered all hosts",
                                 onfail="ONOS incorrectly discovered hosts on nodes: " +
                                 str( hostFails ) )
        topoResults = hostsResults and linksResults and devicesResults
        utilities.assert_equals( expect=main.TRUE,
                                 actual=topoResults,
                                 onpass="ONOS correctly discovered the topology",
                                 onfail="ONOS incorrectly discovered the topology" )
        return topoResults

    def ping( self, srcList, dstList, ipv6=False, expect=True, wait=1, acceptableFailed=0, collectT3=True, t3Simple=False ):
        """
        Description:
            Ping from every host in srcList to every host in dstList and
            verify if ping results are as expected.
            Pings are executed in parallel from host components
        Options:
            src: a list of source host names, e.g. [ "h1", "h2" ]
            dst: a list of destination host names, e.g. [ "h3", "h4" ]
            expect: expect ping result to pass if True, otherwise fail
            acceptableFailed: maximum number of failed pings acceptable for
                              each src-dst host pair
            collectT3: save t3-troubleshoot output for src and dst host that failed to ping
            t3Simple: use t3-troubleshoot-simple command when collecting t3 output
        Returns:
            main.TRUE if all ping results are expected, otherwise main.FALSE
        """
        main.log.info( "Pinging from {} to {}, expected result is {}".format( srcList, dstList,
                                                                              "pass" if expect else "fail" ) )
        # Verify host component has been created
        srcIpList = {}
        for src in srcList:
            if not hasattr( main, src ):
                main.log.info( "Creating component for host {}".format( src ) )
                main.Network.createHostComponent( src )
                hostHandle = getattr( main, src )
                if hasattr( main, 'Mininet1' ):
                    main.log.info( "Starting CLI on host {}".format( src ) )
                    hostHandle.startHostCli()
                else:
                    hostHandle.connectInband()
            srcIpList[ src ] = main.Network.getIPAddress( src, proto='IPV6' if ipv6 else 'IPV4' )
        unexpectedPings = []
        for dst in dstList:
            dstIp = main.Network.getIPAddress( dst, proto='IPV6' if ipv6 else 'IPV4' )
            # Start pings from src hosts in parallel
            pool = []
            for src in srcList:
                srcIp = srcIpList[ src ]
                if not srcIp or not dstIp:
                    if expect:
                        unexpectedPings.append( [ src, dst, "no IP" ] )
                    continue
                if srcIp == dstIp:
                    continue
                hostHandle = getattr( main, src )
                thread = main.Thread( target=utilities.retry,
                                      name="{}-{}".format( srcIp, dstIp ),
                                      args=[ hostHandle.pingHostSetAlternative, [ main.FALSE ] ],
                                      kwargs={ "args": ( [ dstIp ], wait, ipv6 ),
                                               "attempts": acceptableFailed + 1,
                                               "sleep": 1 } )
                pool.append( thread )
                thread.start()
            # Wait for threads to finish and check ping result
            for thread in pool:
                thread.join( 10 )
                srcIp, dstIp = thread.name.split( "-" )
                if expect and not thread.result or not expect and thread.result:
                    unexpectedPings.append( [ srcIp, dstIp, "fail" if expect else "pass" ] )
        main.log.info( "Unexpected pings: {}".format( unexpectedPings ) )
        if collectT3:
            for unexpectedPing in unexpectedPings:
                if unexpectedPing[ 2 ] == "no IP":
                    continue
                srcIp = unexpectedPing[ 0 ]
                dstIp = unexpectedPing[ 1 ]
                main.log.debug( "Collecting t3 with source {} and destination {}".format( srcIp, dstIp ) )
                cmdList = main.Cluster.active( 0 ).CLI.composeT3Command( srcIp, dstIp, ipv6, True, t3Simple )
                if not cmdList:
                    main.log.warn( "Failed to compose T3 command with source {} and destination {}".format( srcIp, dstIp ) )
                    continue
                for i in range( 0, len( cmdList ) ):
                    cmd = cmdList[ i ]
                    main.log.debug( "t3 command: {}".format( cmd ) )
                    main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, cmd, main.logdir,
                                                "t3-CASE{}-{}-{}-route{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp, i ),
                                                timeout=10,
                                                cliPort=main.Cluster.active(0).CLI.karafPort )
        return main.FALSE if unexpectedPings else main.TRUE

    def sendScapyPackets( self, sender, receiver, pktFilter, pkt, sIface=None, dIface=None, expect=True, acceptableFailed=0, collectT3=True, t3Command="" ):
        """
        Description:
            Send Scapy packets from sender to receiver and verify if result is as expected and retry if neccessary
            If collectT3 is True and t3Command is specified, collect t3-troubleshoot output on unexpected scapy results
        Options:
            sender: the component of the host that is sending packets
            receiver: the component of the host that is receiving packets
            pktFilter: packet filter used by receiver
            pkt: keyword that is expected to be conrained in the received packets
            expect: expect receiver to receive the packet if True, otherwise not receiving the packet
            acceptableFailed: maximum number of unexpected scapy results acceptable
            collectT3: save t3-troubleshoot output for unexpected scapy results
        Returns:
            main.TRUE if scapy result is expected, otherwise main.FALSE
        Note: It is assumed that Scapy is already started on the destination host
        """
        main.log.info( "Sending scapy packets from  {} to {}, expected result is {}".format( sender.name, receiver.name,
                                                                                             "pass" if expect else "fail" ) )
        scapyResult = utilities.retry( self.sendScapyPacketsHelper,
                                       main.FALSE,
                                       args=( sender, receiver, pktFilter, pkt,
                                              sIface, dIface, expect ),
                                       attempts=acceptableFailed + 1,
                                       sleep=1 )
        if not scapyResult and collectT3 and t3Command:
            main.log.debug( "Collecting t3 with source {} and destination {}".format( sender.name, receiver.name ) )
            main.log.debug( "t3 command: {}".format( t3Command ) )
            main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, t3Command, main.logdir,
                                        "t3-CASE{}-{}-{}-".format( main.CurrentTestCaseNumber, sender.name, receiver.name ),
                                        cliPort=main.Cluster.active(0).CLI.karafPort )
        return scapyResult

    def sendScapyPacketsHelper( self, sender, receiver, pktFilter, pkt, sIface=None, dIface=None, expect=True ):
        """
        Description:
            Send Scapy packets from sender to receiver and verify if result is as expected
        Options:
            sender: the component of the host that is sending packets
            receiver: the component of the host that is receiving packets
            pktFilter: packet filter used by receiver
            pkt: keyword that is expected to be conrained in the received packets
            expect: expect receiver to receive the packet if True, otherwise not receiving the packet
        Returns:
            main.TRUE if scapy result is expected, otherwise main.FALSE
        """
        started = receiver.startFilter( ifaceName=dIface, pktFilter=pktFilter )
        if not started:
            main.log.error("Failed to start Scapy packet filter")
            return main.FALSE
        sender.sendPacket( iface=sIface )
        finished = receiver.checkFilter()
        packet = ""
        if finished:
            packets = receiver.readPackets( detailed=True )
            for packet in packets.splitlines():
                main.log.debug( packet )
        else:
            kill = receiver.killFilter()
            main.log.debug( kill )
            sender.handle.sendline( "" )
            sender.handle.expect( sender.scapyPrompt )
            main.log.debug( sender.handle.before )
        packetCaptured = True if pkt in packet else False
        return main.TRUE if packetCaptured == expect else main.FALSE

    def pingAndCapture( self, srcHost, dstIp, dstHost, dstIntf, ipv6=False, expect=True, acceptableFailed=0, collectT3=True, t3Simple=False ):
        """
        Description:
            Ping from src host to dst IP and capture packets at dst Host using Scapy and retry if neccessary
            If collectT3 is True, collect t3-troubleshoot output on unexpected scapy results
        Options:
            srcHost: name of the source host
            dstIp: destination IP of the ping packets
            dstHost: host that runs Scapy to capture the packets
            dstIntf: name of the interface on the destination host
            ipv6: ping with IPv6 if True; Otherwise IPv4
            expect: use True if the ping is expected to be captured at destination;
                    Otherwise False
            acceptableFailed: maximum number of failed pings acceptable
            collectT3: save t3-troubleshoot output for src and dst host that failed to ping
            t3Simple: use t3-troubleshoot-simple command when collecting t3 output
        Returns:
            main.TRUE if packet capturing result is expected, otherwise main.FALSE
        Note: It is assumed that Scapy is already started on the destination host
        """
        main.log.info( "Pinging from {} to {}, expected {} capture the packet at {}".format( srcHost, dstIp,
                       "to" if expect else "not to", dstHost ) )
        # Verify host component has been created
        if not hasattr( main, srcHost ):
            main.log.info( "Creating component for host {}".format( srcHost ) )
            main.Network.createHostComponent( srcHost )
            srcHandle = getattr( main, srcHost )
            main.log.info( "Starting CLI on host {}".format( srcHost ) )
            srcHandle.startHostCli()
        trafficResult = utilities.retry( self.pingAndCaptureHelper,
                                         main.FALSE,
                                         args=( srcHost, dstIp, dstHost, dstIntf, ipv6, expect ),
                                         attempts=acceptableFailed + 1,
                                         sleep=1 )
        if not trafficResult and collectT3:
            srcIp = main.Network.getIPAddress( srcHost, proto='IPV6' if ipv6 else 'IPV4' )
            main.log.debug( "Collecting t3 with source {} and destination {}".format( srcIp, dstIp ) )
            cmdList = main.Cluster.active( 0 ).CLI.composeT3Command( srcIp, dstIp, ipv6, True, t3Simple )
            if not cmdList:
                main.log.warn( "Failed to compose T3 command with source {} and destination {}".format( srcIp, dstIp ) )
            for i in range( 0, len( cmdList ) ):
                cmd = cmdList[ i ]
                main.log.debug( "t3 command: {}".format( cmd ) )
                main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, cmd, main.logdir,
                                            "t3-CASE{}-{}-{}-route{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp, i ),
                                            timeout=10,
                                            cliPort=main.Cluster.active(0).CLI.karafPort )
        return trafficResult

    def pingAndCaptureHelper( self, srcHost, dstIp, dstHost, dstIntf, ipv6=False, expect=True ):
        """
        Description:
            Ping from src host to dst IP and capture packets at dst Host using Scapy
        Options:
            srcHost: name of the source host
            dstIp: destination IP of the ping packets
            dstHost: host that runs Scapy to capture the packets
            dstIntf: name of the interface on the destination host
            ipv6: ping with IPv6 if True; Otherwise IPv4
            expect: use True if the ping is expected to be captured at destination;
                    Otherwise False
        Returns:
            main.TRUE if packet capturing result is expected, otherwise main.FALSE
        """
        packetCaptured = True
        srcHandle = getattr( main, srcHost )
        dstHandle = getattr( main, dstHost )
        pktFilter = "ip6 host {}".format( dstIp ) if ipv6 else "ip host {}".format( dstIp )
        started = dstHandle.startFilter( ifaceName=dstIntf, pktFilter=pktFilter )
        if not started:
            main.log.error("Failed to start Scapy packet filter")
            return main.FALSE
        srcHandle.pingHostSetAlternative( [ dstIp ], IPv6=ipv6 )
        finished = dstHandle.checkFilter()
        packet = ""
        if finished:
            packets = receiver.readPackets( detailed=True )
            for packet in packets.splitlines():
                main.log.debug( packet )
        else:
            kill = dstHandle.killFilter()
            main.log.debug( kill )
        packetCaptured = True if "dst={}".format( dstIp ) in packet else False
        return main.TRUE if packetCaptured == expect else main.FALSE