TestON/tests/DEMO_SDNIP/DEMO_SDNIP.py

# Testing the functionality of SDN-IP with single ONOS instance
class DEMO_SDNIP:

    def __init__( self ):
        self.default = ''
        global branchName

    def CASE100( self, main ):
        """
            Start mininet
        """
        import imp
        main.case( "Setup the Mininet testbed" )
        main.log.demoSummary( "DEMO:Mininet: Setup the Mininet testbed" )
        main.dependencyPath = main.testDir + \
                              main.params[ 'DEPENDENCY' ][ 'path' ]
        main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]

        main.step( "Starting Mininet Topology" )
        topology = main.dependencyPath + main.topology
        topoResult = main.Mininet.startNet( topoFile=topology )
        utilities.assert_equals( expect=main.TRUE,
                                 actual=topoResult,
                                 onpass="Successfully loaded topology",
                                 onfail="Failed to load topology" )
        # Exit if topology did not load properly
        if not topoResult:
            main.cleanup()
            main.exit()
        main.step( "Connect switches to controllers" )
        main.log.demoSummary( "DEMO:Mininet: Connecting switches to controllers" )

        # connect all switches to controllers
        swResult = main.TRUE
        for i in range ( 1, int( main.params['config']['switchNum'] ) + 1 ):
            sw = "sw%s" % ( i )
            swResult = swResult and main.Mininet.assignSwController( sw,
                                                 [ONOS1Ip, ONOS2Ip, ONOS3Ip] )
        utilities.assert_equals( expect=main.TRUE,
                                 actual=swResult,
                                 onpass="Successfully connect all switches to ONOS",
                                 onfail="Failed to connect all switches to ONOS" )
        if not swResult:
            main.cleanup()
            main.exit()


    def CASE101( self, main ):
        """
           Package ONOS and install it
           Startup sequence:
           cell <name>
           onos-verify-cell
           onos-package
           onos-install -f
           onos-wait-for-start
        """
        import json
        import time
        import os
        from operator import eq

        main.case( "Setting up ONOS environment" )
        main.log.demoSummary( "DEMO: TestStation: Setting up ONOS environment" )

        cellName = main.params[ 'ENV' ][ 'cellName' ]
        global ONOS1Ip
        global ONOS2Ip
        global ONOS3Ip
        ONOS1Ip = os.getenv( main.params[ 'CTRL' ][ 'ip1' ] )
        ONOS2Ip = os.getenv( main.params[ 'CTRL' ][ 'ip2' ] )
        ONOS3Ip = os.getenv( main.params[ 'CTRL' ][ 'ip3' ] )
        ipList = [ ONOS1Ip, ONOS2Ip, ONOS3Ip ]

        global peer64514
        global peer64515
        global peer64516
        peer64514 = main.params['config']['peer64514']
        peer64515 = main.params['config']['peer64515']
        peer64516 = main.params['config']['peer64516']

        main.step( "Applying cell variable to environment" )
        cellAppString = main.params[ 'ENV' ][ 'appString' ]
        main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
                                       main.Mininet.ip_address,
                                       cellAppString, ipList )

        cellResult = main.ONOSbench.setCell( cellName )
        utilities.assert_equals( expect=main.TRUE,
                                 actual=cellResult,
                                 onpass="Set cell succeeded",
                                 onfail="Set cell failed" )

        verifyResult = main.ONOSbench.verifyCell()
        utilities.assert_equals( expect=main.TRUE,
                                 actual=verifyResult,
                                 onpass="Verify cell succeeded",
                                 onfail="Verify cell failed" )

        main.log.demoSummary( "DEMO: ONOS: Connecting to ONOS" )
        '''
        p = main.ONOSbench.handle
        p.sendline( "stc setup" )
        p.expect( "\$", timeout=180 )
        # TODO: add assert here after converting to a function
        '''

        main.step( "Checking if ONOS CLI is ready to start" )
        main.CLIs = []
        main.log.demoSummary( "DEMO: ONOS1: Connecting to ONOS1" )
        cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip,
                commandlineTimeout=100, onosStartTimeout=600 )
        main.CLIs.append( main.ONOScli1 )
        main.log.demoSummary( "DEMO: ONOS2: Connecting to ONOS2" )
        cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip,
                commandlineTimeout=100, onosStartTimeout=600 )
        main.CLIs.append( main.ONOScli2 )
        main.log.demoSummary( "DEMO: ONOS3: Connecting to ONOS3" )
        cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip,
                commandlineTimeout=100, onosStartTimeout=600 )
        main.CLIs.append( main.ONOScli3 )
        cliResult = cliResult1 and cliResult2 and cliResult3
        utilities.assert_equals( expect=main.TRUE,
                                 actual=cliResult,
                                 onpass="ONOS CLI is ready",
                                 onfail="ONOS CLI is not ready" )

        main.step( "Checking if ONOS CLI is ready for issuing commands" )
        main.log.demoSummary( "DEMO: ONOS: Checking CLI" )
        for i in range( 10 ):
            ready = True
            for cli in main.CLIs:
                output = cli.summary()
                if not output:
                    ready = False
            if ready:
                break
            time.sleep( 30 )
        utilities.assert_equals( expect=True, actual=ready,
                                 onpass="ONOS summary command succeded",
                                 onfail="ONOS summary command failed" )

        if not ready:
            main.log.error( "ONOS startup failed!" )
            main.cleanup()
            main.exit()
        time.sleep( int( main.params['timers']['Readability'] ) )

    def CASE199( self, main ):
        main.case( "Verify topology discovery" )
        main.log.info( "Waiting for link discovery......" )
        time.sleep( int( main.params['timers']['TopoDiscovery'] ) )

        main.step( "Get links in the network" )
        summaryResult = main.ONOScli1.summary()
        linkNum = json.loads( summaryResult )[ "links" ]
        main.log.info( "Expected 100 links, actual number is: {}".format( linkNum ) )
        if linkNum < 100:
            main.log.error( "Link number is wrong! Retrying..." )
            time.sleep( int( main.params['timers']['TopoDiscovery'] ) )
            summaryResult = main.ONOScli1.summary()
            linkNum = json.loads( summaryResult )[ "links" ]
        main.log.info( "Expected 100 links, actual number is: {}".format( linkNum ) )
        utilities.assert_equals( expect=100,
                                 actual=linkNum,
                                 onpass="ONOS correctly discovered all links",
                                 onfail="ONOS Failed to discover all links" )
        if linkNum < 100:
            main.cleanup()
            main.exit()
        time.sleep( int( main.params['timers']['Readability'] ) )

    def CASE200( self, main ):
        main.case( "Activate sdn-ip application" )

        main.step( "Activate sdn-ip application" )
        main.log.demoSummary( "DEMO:ONOS1: Activate sdn-ip application" )
        activeSDNIPresult = main.ONOScli1.activateApp( "org.onosproject.sdnip" )
        utilities.assert_equals( expect=main.TRUE,
                                 actual=activeSDNIPresult,
                                 onpass="Activate SDN-IP succeeded",
                                 onfail="Activate SDN-IP failed" )
        if not activeSDNIPresult:
            main.log.info( "Activate SDN-IP failed!" )
            main.cleanup()
            main.exit()
        time.sleep( int( main.params['timers']['Readability'] ) )


    def CASE102( self, main ):
        '''
        This test case is to load the methods from other Python files, and create
        tunnels from mininet host to onos nodes.
        '''
        import time
        main.case( "Create tunnels between Quagga and SDNIP Application" )
        # load the methods from other file
        wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
        main.Functions = imp.load_source( wrapperFile1,
                                          main.dependencyPath +
                                          wrapperFile1 +
                                          ".py" )
        # Create tunnels
        main.log.demoSummary( "DEMO:Mininet: Creating tunnels between bgp speakers" )
        # ONOS1
        main.Functions.setupTunnel( main, '1.1.1.2', 2000, ONOS1Ip, 2000 )
        # ONOS2
        main.Functions.setupTunnel( main, '1.1.1.4', 2000, ONOS2Ip, 2000 )
        # ONOS3
        main.Functions.setupTunnel( main, '1.1.1.6', 2000, ONOS3Ip, 2000 )

        time.sleep( int( main.params[ 'timers' ][ 'SdnIpSetup' ] ) )


    def CASE1( self, main ):
        '''
        ping test from 3 bgp peers to BGP speaker
        '''

        main.case( "Ping between BGP peers and speakers" )
        main.log.demoSummary( "DEMO:Mininet: Ping between bgp peers and speakers" )
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker1"],
                       peers=["peer64514", "peer64515", "peer64516"],
                       expectAllSuccess=True )
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker2"],
                       peers=[peer64514, peer64515, peer64516],
                       expectAllSuccess=True )
        time.sleep( int( main.params['timers']['Readability'] ) )

    def CASE2( self, main ):
        '''
        point-to-point intents test for each BGP peer and BGP speaker pair
        '''
        import time
        main.case( "Check point-to-point intents" )
        main.log.demoSummary( "DEMO:ONOS1: Verify Point-to-Point intents" )
        main.log.info( "There are %s BGP peers in total "
                       % main.params[ 'config' ][ 'peerNum' ] )
        main.step( "Check P2P intents number from ONOS CLI" )

        getIntentsResult = main.ONOScli1.intents( jsonFormat=True )
        bgpIntentsActualNum = \
            main.QuaggaCliSpeaker1.extractActualBgpIntentNum( getIntentsResult )
        bgpIntentsExpectedNum = int( main.params[ 'config' ][ 'peerNum' ] ) * 6 * 2
        if bgpIntentsActualNum != bgpIntentsExpectedNum:
            time.sleep( int( main.params['timers']['RouteDelivery'] ) )
            getIntentsResult = main.ONOScli1.intents( jsonFormat=True )
            bgpIntentsActualNum = \
                main.QuaggaCliSpeaker1.extractActualBgpIntentNum( getIntentsResult )
        main.log.info( "bgpIntentsExpected num is:" )
        main.log.info( bgpIntentsExpectedNum )
        main.log.info( "bgpIntentsActual num is:" )
        main.log.info( bgpIntentsActualNum )
        utilities.assert_equals( \
            expect=bgpIntentsExpectedNum,
            actual=bgpIntentsActualNum,
            onpass="PointToPointIntent Intent Num is correct!",
            onfail="PointToPointIntent Intent Num is wrong!" )
        time.sleep( int( main.params['timers']['Readability'] ) )

    def CASE3( self, main ):
        '''
        routes and intents check to all BGP peers
        '''
        import time
        main.case( "Check routes and M2S intents to all BGP peers" )

        main.step( "Check routes installed" )
        allRoutesExpected = []
        allRoutesExpected.append( "4.0.0.0/24" + "/" + "10.0.4.1" )
        allRoutesExpected.append( "5.0.0.0/24" + "/" + "10.0.5.1" )
        allRoutesExpected.append( "6.0.0.0/24" + "/" + "10.0.6.1" )

        main.log.demoSummary( "DEMO:Quagga: Check BGP routes" )
        getRoutesResult = main.ONOScli1.routes( jsonFormat=True )
        allRoutesActual = \
            main.QuaggaCliSpeaker1.extractActualRoutesMaster( getRoutesResult )
        allRoutesStrExpected = str( sorted( allRoutesExpected ) )
        allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )
        if allRoutesStrActual != allRoutesStrExpected:
            time.sleep( int( main.params['timers']['RouteDelivery'] ) )
            allRoutesActual = \
                main.QuaggaCliSpeaker1.extractActualRoutesMaster( getRoutesResult )
            allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )

        main.log.info( "Routes expected:" )
        main.log.info( allRoutesStrExpected )
        main.log.info( "Routes get from ONOS CLI:" )
        main.log.info( allRoutesStrActual )
        utilities.assert_equals( \
            expect=allRoutesStrExpected, actual=allRoutesStrActual,
            onpass="Routes are correct!",
            onfail="Routes are wrong!" )
        time.sleep( int( main.params['timers']['Readability'] ) )

        main.step( "Check M2S intents installed" )
        main.log.demoSummary( "DEMO:ONOS1: Check Multi-point intents" )
        getIntentsResult = main.ONOScli1.intents( jsonFormat=True )
        routeIntentsActualNum = \
            main.QuaggaCliSpeaker1.extractActualRouteIntentNum( getIntentsResult )
        routeIntentsExpectedNum = 3
        if routeIntentsActualNum != routeIntentsExpectedNum:
            time.sleep( int( main.params['timers']['RouteDelivery'] ) )
            getIntentsResult = main.ONOScli1.intents( jsonFormat=True )
            routeIntentsActualNum = \
                main.QuaggaCliSpeaker1.extractActualRouteIntentNum( getIntentsResult )

        main.log.info( "MultiPointToSinglePoint Intent Num expected is:" )
        main.log.info( routeIntentsExpectedNum )
        main.log.info( "MultiPointToSinglePoint Intent NUM Actual is:" )
        main.log.info( routeIntentsActualNum )
        utilities.assert_equals( \
            expect=routeIntentsExpectedNum,
            actual=routeIntentsActualNum,
            onpass="MultiPointToSinglePoint Intent Num is correct!",
            onfail="MultiPointToSinglePoint Intent Num is wrong!" )
        time.sleep( int( main.params['timers']['Readability'] ) )

        main.log.demoSummary( "DEMO:ONOS1: Check flows" )
        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                     main.FALSE,
                                     kwargs={'isPENDING':False},
                                     attempts=10 )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=flowCheck,
            onpass="Flow status is correct!",
            onfail="Flow status is wrong!" )
        time.sleep( int( main.params['timers']['Readability'] ) )


    def CASE4( self, main ):
        '''
        Ping test in data plane for each route
        '''
        main.case( "Ping test for each route, all hosts behind BGP peers" )
        main.log.demoSummary( "DEMO:Mininet: Pinging across BGP routes" )
        main.Functions.pingHostToHost( main,
                        hosts=["host64514", "host64515", "host64516"],
                        expectAllSuccess=True )
        time.sleep( int( main.params['timers']['Readability'] ) )
        #main.log.demoSummary( "DEMO: STOP" )  #FIXME For testing

    def CASE5( self, main ):
        '''
        Cut links to peers one by one, check routes/intents
        '''
        import time
        main.case( "Bring down links and check routes/intents" )
        main.log.demoSummary( "DEMO:Mininet: Bringing down links between Quagga and the network" )
        main.step( "Bring down the link between sw32 and peer64514" )
        linkResult1 = main.Mininet.link( END1="sw32", END2="peer64514",
                                         OPTION="down" )
        utilities.assert_equals( expect=main.TRUE,
                                actual=linkResult1,
                                onpass="Bring down link succeeded!",
                                onfail="Bring down link failed!" )

        if linkResult1 == main.TRUE:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.log.debug( main.ONOScli1.links() )  #FIXME: DEBUG
            main.Functions.checkRouteNum( main, 2 )
            main.Functions.checkM2SintentNum( main, 2 )
        else:
            main.log.error( "Bring down link failed!" )
            main.cleanup()
            main.exit()

        main.step( "Bring down the link between sw8 and peer64515" )
        linkResult2 = main.Mininet.link( END1="sw8", END2="peer64515",
                                         OPTION="down" )
        utilities.assert_equals( expect=main.TRUE,
                                actual=linkResult2,
                                onpass="Bring down link succeeded!",
                                onfail="Bring down link failed!" )
        if linkResult2 == main.TRUE:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.Functions.checkRouteNum( main, 1 )
            main.Functions.checkM2SintentNum( main, 1 )
        else:
            main.log.error( "Bring down link failed!" )
            main.cleanup()
            main.exit()

        main.step( "Bring down the link between sw28 and peer64516" )
        linkResult3 = main.Mininet.link( END1="sw28", END2="peer64516",
                                         OPTION="down" )
        utilities.assert_equals( expect=main.TRUE,
                                actual=linkResult3,
                                onpass="Bring down link succeeded!",
                                onfail="Bring down link failed!" )
        if linkResult3 == main.TRUE:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.Functions.checkRouteNum( main, 0 )
            main.Functions.checkM2SintentNum( main, 0 )
        else:
            main.log.error( "Bring down link failed!" )
            main.cleanup()
            main.exit()

        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                     main.FALSE,
                                     kwargs={'isPENDING':False},
                                     attempts=10 )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=flowCheck,
            onpass="Flow status is correct!",
            onfail="Flow status is wrong!" )

        # Ping test
        main.log.demoSummary( "DEMO:Mininet: Verify lost connectivity across routes" )
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker1"],
                       peers=["peer64514", "peer64515", "peer64516"],
                       expectAllSuccess=False )
        main.Functions.pingHostToHost( main,
                        hosts=["host64514", "host64515", "host64516"],
                        expectAllSuccess=False )


    def CASE6( self, main ):
        '''
        Recover links to peers one by one, check routes/intents
        '''
        import time
        main.case( "Bring up links and check routes/intents" )
        main.step( "Bring up the link between sw32 and peer64514" )
        main.log.demoSummary( "DEMO:Mininet: Bring back up links between Quagga and the network" )
        linkResult1 = main.Mininet.link( END1="sw32", END2="peer64514",
                                         OPTION="up" )
        utilities.assert_equals( expect=main.TRUE,
                                actual=linkResult1,
                                onpass="Bring up link succeeded!",
                                onfail="Bring up link failed!" )
        if linkResult1 == main.TRUE:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.Functions.checkRouteNum( main, 1 )
            main.Functions.checkM2SintentNum( main, 1 )
        else:
            main.log.error( "Bring up link failed!" )
            main.cleanup()
            main.exit()

        main.step( "Bring up the link between sw8 and peer64515" )
        linkResult2 = main.Mininet.link( END1="sw8", END2="peer64515",
                                         OPTION="up" )
        utilities.assert_equals( expect=main.TRUE,
                                actual=linkResult2,
                                onpass="Bring up link succeeded!",
                                onfail="Bring up link failed!" )
        if linkResult2 == main.TRUE:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.Functions.checkRouteNum( main, 2 )
            main.Functions.checkM2SintentNum( main, 2 )
        else:
            main.log.error( "Bring up link failed!" )
            main.cleanup()
            main.exit()

        main.step( "Bring up the link between sw28 and peer64516" )
        linkResult3 = main.Mininet.link( END1="sw28", END2="peer64516",
                                         OPTION="up" )
        utilities.assert_equals( expect=main.TRUE,
                                actual=linkResult3,
                                onpass="Bring up link succeeded!",
                                onfail="Bring up link failed!" )
        if linkResult3 == main.TRUE:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.Functions.checkRouteNum( main, 3 )
            main.Functions.checkM2SintentNum( main, 3 )
        else:
            main.log.error( "Bring up link failed!" )
            main.cleanup()
            main.exit()

        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                     main.FALSE,
                                     kwargs={'isPENDING':False},
                                     attempts=10 )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=flowCheck,
            onpass="Flow status is correct!",
            onfail="Flow status is wrong!" )

        # Ping test
        main.log.demoSummary( "DEMO:Mininet: Verify connectivity is restored across routes" )
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker1"],
                       peers=["peer64514", "peer64515", "peer64516"],
                       expectAllSuccess=True )
        main.Functions.pingHostToHost( main,
                        hosts=["host64514", "host64515", "host64516"],
                        expectAllSuccess=True )


    def CASE7( self, main ):
        '''
        Shut down a edge switch, check P-2-P and M-2-S intents, ping test
        '''
        import time
        main.case( "Stop edge sw32,check P-2-P and M-2-S intents, ping test" )
        main.step( "Stop sw32" )
        result = main.Mininet.switch( SW="sw32", OPTION="stop" )
        utilities.assert_equals( expect=main.TRUE, actual=result,
                                onpass="Stopping switch succeeded!",
                                onfail="Stopping switch failed!" )

        if result == main.TRUE:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.Functions.checkRouteNum( main, 2 )
            main.Functions.checkM2SintentNum( main, 2 )
            main.Functions.checkP2PintentNum( main, 12 * 2 )
        else:
            main.log.error( "Stopping switch failed!" )
            main.cleanup()
            main.exit()

        main.step( "Check ping between hosts behind BGP peers" )
        result1 = main.Mininet.pingHost( src="host64514", target="host64515" )
        result2 = main.Mininet.pingHost( src="host64515", target="host64516" )
        result3 = main.Mininet.pingHost( src="host64514", target="host64516" )

        pingResult1 = ( result1 == main.FALSE ) and ( result2 == main.TRUE ) \
                      and ( result3 == main.FALSE )
        utilities.assert_equals( expect=True, actual=pingResult1,
                                 onpass="Ping test result is correct",
                                 onfail="Ping test result is wrong" )

        if pingResult1 == False:
            main.cleanup()
            main.exit()

        main.step( "Check ping between BGP peers and speaker1" )
        result4 = main.Mininet.pingHost( src="speaker1", target="peer64514" )
        result5 = main.Mininet.pingHost( src="speaker1", target="peer64515" )
        result6 = main.Mininet.pingHost( src="speaker1", target="peer64516" )

        pingResult2 = ( result4 == main.FALSE ) and ( result5 == main.TRUE ) \
                      and ( result6 == main.TRUE )
        utilities.assert_equals( expect=True, actual=pingResult2,
                                 onpass="Speaker1 ping peers successful",
                                 onfail="Speaker1 ping peers NOT successful" )

        if pingResult2 == False:
            main.cleanup()
            main.exit()

        main.step( "Check ping between BGP peers and speaker2" )
        # TODO
        result7 = main.Mininet.pingHost( src="speaker2", target=peer64514 )
        result8 = main.Mininet.pingHost( src="speaker2", target=peer64515 )
        result9 = main.Mininet.pingHost( src="speaker2", target=peer64516 )

        pingResult3 = ( result7 == main.FALSE ) and ( result8 == main.TRUE ) \
                                                and ( result9 == main.TRUE )
        utilities.assert_equals( expect=True, actual=pingResult2,
                                 onpass="Speaker2 ping peers successful",
                                 onfail="Speaker2 ping peers NOT successful" )

        if pingResult3 == False:
            main.cleanup()
            main.exit()

        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                     main.FALSE,
                                     kwargs={'isPENDING':False},
                                     attempts=10 )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=flowCheck,
            onpass="Flow status is correct!",
            onfail="Flow status is wrong!" )


    def CASE8( self, main ):
        '''
        Bring up the edge switch (sw32) which was shut down in CASE7,
        check P-2-P and M-2-S intents, ping test
        '''
        import time
        main.case( "Start the edge sw32, check P-2-P and M-2-S intents, ping test" )
        main.step( "Start sw32" )
        result1 = main.Mininet.switch( SW="sw32", OPTION="start" )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=result1,
            onpass="Starting switch succeeded!",
            onfail="Starting switch failed!" )

        result2 = main.Mininet.assignSwController( "sw32", ONOS1Ip )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=result2,
            onpass="Connect switch to ONOS succeeded!",
            onfail="Connect switch to ONOS failed!" )

        if result1 and result2:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.Functions.checkRouteNum( main, 3 )
            main.Functions.checkM2SintentNum( main, 3 )
            main.Functions.checkP2PintentNum( main, 18 * 2 )
        else:
            main.log.error( "Starting switch failed!" )
            main.cleanup()
            main.exit()

        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                     main.FALSE,
                                     kwargs={'isPENDING':False},
                                     attempts=10 )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=flowCheck,
            onpass="Flow status is correct!",
            onfail="Flow status is wrong!" )

        # Ping test
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker1"],
                       peers=["peer64514", "peer64515", "peer64516"],
                       expectAllSuccess=True )
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker2"],
                       peers=[peer64514, peer64515, peer64516],
                       expectAllSuccess=True )
        main.Functions.pingHostToHost( main,
                        hosts=["host64514", "host64515", "host64516"],
                        expectAllSuccess=True )


    def CASE9( self, main ):
        '''
        Bring down a switch in best path, check:
        route number, P2P intent number, M2S intent number, ping test
        '''
        main.case( "Stop sw11 located in best path, \
        check route number, P2P intent number, M2S intent number, ping test" )

        main.log.info( "Check the flow number correctness before stopping sw11" )
        main.Functions.checkFlowNum( main, "sw11", 19 )
        main.Functions.checkFlowNum( main, "sw1", 3 )
        main.Functions.checkFlowNum( main, "sw7", 3 )

        main.step( "Stop sw11" )
        result = main.Mininet.switch( SW="sw11", OPTION="stop" )
        utilities.assert_equals( expect=main.TRUE, actual=result,
                                onpass="Stopping switch succeeded!",
                                onfail="Stopping switch failed!" )
        if result:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.Functions.checkRouteNum( main, 3 )
            main.Functions.checkM2SintentNum( main, 3 )
            main.Functions.checkP2PintentNum( main, 18 * 2 )
        else:
            main.log.error( "Stopping switch failed!" )
            main.cleanup()
            main.exit()

        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                     main.FALSE,
                                     kwargs={'isPENDING':False},
                                     attempts=10 )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=flowCheck,
            onpass="Flow status is correct!",
            onfail="Flow status is wrong!" )
        # Ping test
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker1"],
                       peers=["peer64514", "peer64515", "peer64516"],
                       expectAllSuccess=True )
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker2"],
                       peers=[peer64514, peer64515, peer64516],
                       expectAllSuccess=True )
        main.Functions.pingHostToHost( main,
                        hosts=["host64514", "host64515", "host64516"],
                        expectAllSuccess=True )


    def CASE10( self, main ):
        '''
        Bring up the switch which was stopped in CASE9, check:
        route number, P2P intent number, M2S intent number, ping test
        '''
        main.case( "Start sw11 which was stopped in CASE9, \
        check route number, P2P intent number, M2S intent number, ping test" )

        main.log.info( "Check the flow status before starting sw11" )
        main.Functions.checkFlowNum( main, "sw1", 17 )
        main.Functions.checkFlowNum( main, "sw7", 5 )

        main.step( "Start sw11" )
        result1 = main.Mininet.switch( SW="sw11", OPTION="start" )
        utilities.assert_equals( expect=main.TRUE, actual=result1,
                                onpass="Starting switch succeeded!",
                                onfail="Starting switch failed!" )
        result2 = main.Mininet.assignSwController( "sw11", ONOS1Ip )
        utilities.assert_equals( expect=main.TRUE, actual=result2,
                                onpass="Connect switch to ONOS succeeded!",
                                onfail="Connect switch to ONOS failed!" )
        if result1 and result2:
            time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
            main.Functions.checkRouteNum( main, 3 )
            main.Functions.checkM2SintentNum( main, 3 )
            main.Functions.checkP2PintentNum( main, 18 * 2 )

        else:
            main.log.error( "Starting switch failed!" )
            main.cleanup()
            main.exit()

        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                     main.FALSE,
                                     kwargs={'isPENDING':False},
                                     attempts=10 )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=flowCheck,
            onpass="Flow status is correct!",
            onfail="Flow status is wrong!" )
        # Ping test
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker1"],
                       peers=["peer64514", "peer64515", "peer64516"],
                       expectAllSuccess=True )
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker2"],
                       peers=[peer64514, peer64515, peer64516],
                       expectAllSuccess=True )
        main.Functions.pingHostToHost( main,
                        hosts=["host64514", "host64515", "host64516"],
                        expectAllSuccess=True )


    def CASE11(self, main):
        import time
        main.case( "Kill speaker1, check:\
        route number, P2P intent number, M2S intent number, ping test" )
        main.log.info( "Check network status before killing speaker1" )
        main.Functions.checkRouteNum( main, 3 )
        main.Functions.checkM2SintentNum( main, 3 )
        main.Functions.checkP2PintentNum( main, 18 * 2 )
        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                     main.FALSE,
                                     kwargs={'isPENDING':False},
                                     attempts=10 )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=flowCheck,
            onpass="Flow status is correct!",
            onfail="Flow status is wrong!" )

        main.Functions.pingSpeakerToPeer( main, speakers=["speaker1"],
                       peers=["peer64514", "peer64515", "peer64516"],
                       expectAllSuccess=True )
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker2"],
                       peers=[peer64514, peer64515, peer64516],
                       expectAllSuccess=True )
        main.Functions.pingHostToHost( main,
                        hosts=["host64514", "host64515", "host64516"],
                        expectAllSuccess=True )

        main.step( "Kill speaker1" )
        command1 = "ps -e | grep bgp -c"
        result1 = main.Mininet.node( "root", command1 )

        # The total BGP daemon number in this test environment is 5.
        if "5" in result1:
            main.log.debug( "Before kill speaker1, 5 BGP daemons - correct" )
        else:
            main.log.warn( "Before kill speaker1, number of BGP daemons is wrong" )
            main.log.info( result1 )

        command2 = "sudo kill -9 `ps -ef | grep quagga-sdn.conf | grep -v grep | awk '{print $2}'`"
        result2 = main.Mininet.node( "root", command2 )

        result3 = main.Mininet.node( "root", command1 )

        utilities.assert_equals( expect=True,
                                 actual=( "4" in result3 ),
                                 onpass="Kill speaker1 succeeded",
                                 onfail="Kill speaker1 failed" )
        if ( "4" not in result3 ) :
            main.log.info( result3 )
            main.cleanup()
            main.exit()

        time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )
        main.Functions.checkRouteNum( main, 3 )
        main.Functions.checkM2SintentNum( main, 3 )
        main.Functions.checkP2PintentNum( main, 18 * 2 )

        main.step( "Check whether all flow status are ADDED" )
        flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                     main.FALSE,
                                     kwargs={'isPENDING':False},
                                     attempts=10 )
        utilities.assert_equals( \
            expect=main.TRUE,
            actual=flowCheck,
            onpass="Flow status is correct!",
            onfail="Flow status is wrong!" )

        '''
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker1"],
                       peers=["peer64514", "peer64515", "peer64516"],
                       expectAllSuccess=False )
        '''
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker2"],
                       peers=[peer64514, peer64515, peer64516],
                       expectAllSuccess=True )
        main.Functions.pingHostToHost( main,
                        hosts=["host64514", "host64515", "host64516"],
                        expectAllSuccess=True )


    def CASE12( self, main ):
        import time
        import json
        main.case( "Bring down leader ONOS node, check: \
        route number, P2P intent number, M2S intent number, ping test" )
        main.step( "Find out ONOS leader node" )
        result = main.ONOScli1.leaders()
        jsonResult = json.loads( result )
        leaderIP = ""
        for entry in jsonResult:
            if entry["topic"] == "org.onosproject.sdnip":
                leaderIP = entry["leader"]
                main.log.info( "leaderIP is: " )
                main.log.info( leaderIP )

        main.step( "Uninstall ONOS/SDN-IP leader node" )
        if leaderIP == ONOS1Ip:
            uninstallResult = main.ONOSbench.onosStop( ONOS1Ip )
        elif leaderIP == ONOS2Ip:
            uninstallResult = main.ONOSbench.onosStop( ONOS2Ip )
        else:
            uninstallResult = main.ONOSbench.onosStop( ONOS3Ip )

        utilities.assert_equals( expect=main.TRUE,
                                 actual=uninstallResult,
                                 onpass="Uninstall ONOS leader succeeded",
                                 onfail="Uninstall ONOS leader failed" )
        if uninstallResult != main.TRUE:
            main.cleanup()
            main.exit()
        time.sleep( int( main.params[ 'timers' ][ 'RouteDelivery' ] ) )

        if leaderIP == ONOS1Ip:
            main.Functions.checkRouteNum( main, 3, ONOScli="ONOScli2" )
            main.Functions.checkM2SintentNum( main, 3, ONOScli="ONOScli2" )
            main.Functions.checkP2PintentNum( main, 18 * 2, ONOScli="ONOScli2" )

            main.step( "Check whether all flow status are ADDED" )
            flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                         main.FALSE,
                                         kwargs={'isPENDING':False},
                                         attempts=10 )
            utilities.assert_equals( \
                expect=main.TRUE,
                actual=flowCheck,
                onpass="Flow status is correct!",
                onfail="Flow status is wrong!" )
        else:
            main.Functions.checkRouteNum( main, 3 )
            main.Functions.checkM2SintentNum( main, 3 )
            main.Functions.checkP2PintentNum( main, 18 * 2 )

            main.step( "Check whether all flow status are ADDED" )
            flowCheck = utilities.retry( main.ONOScli1.checkFlowsState,
                                         main.FALSE,
                                         kwargs={'isPENDING':False},
                                         attempts=10 )
            utilities.assert_equals( \
                expect=main.TRUE,
                actual=flowCheck,
                onpass="Flow status is correct!",
                onfail="Flow status is wrong!" )

        main.Functions.pingSpeakerToPeer( main, speakers=["speaker1"],
                       peers=["peer64514", "peer64515", "peer64516"],
                       expectAllSuccess=True )
        main.Functions.pingSpeakerToPeer( main, speakers=["speaker2"],
                       peers=[peer64514, peer64515, peer64516],
                       expectAllSuccess=True )
        main.Functions.pingHostToHost( main,
                        hosts=["host64514", "host64515", "host64516"],
                        expectAllSuccess=True )

    def CASE1001( self, main):
        '''
        Check onos node log for ERROR
        '''
        import time

        main.case( "Checking onos logs for ERRORs...")
        main.step( "Checking onos logs for ERRORs ...")
        main.log.demoSummary( "DEMO:ONOS: Check onos logs for ERRORs")

        onosErrors={}
        result =  True
        for node in  [ ONOS1Ip, ONOS2Ip, ONOS3Ip ]:
            onosErrors[ node ] = main.ONOSbench.logReport(node, ["ERROR", "WARNING", "EXCEPT"] )
            result  = onosErrors[ node ]  & result

        utilities.assert_equals( \
            expect=main.TRUE,
            actual=result,
            onpass="No onos WARNINGs/ERRORs/Exceptions at the end of the test!",
            onfail="onos has WARNINGs/ERRORs/Exceptions at the end of the test!" )

        time.sleep( int( main.params['timers']['Readability'] ) )