TestON/tests/TopoConvNext/TopoConvNext.py

# TopoPerfNext
#
# Topology Convergence scale-out test for ONOS-next
# NOTE: This test supports up to 7 nodes scale-out scenario
#
# NOTE: Ensure that you have 'tablet.json' file
#      in the onos/tools/package/config directory
# NOTE: You must start this test initially with 3 nodes
#
# andrew@onlab.us

import time
import sys
import os
import re


class TopoConvNext:

    def __init__( self ):
        self.default = ''

    def CASE1( self, main ):
        """
        ONOS startup sequence
        """
        import time

        #******
        # Global cluster count for scale-out purposes
        global cluster_count
        global topo_iteration
        topo_iteration = 1
        cluster_count = 1
        #******
        cell_name = main.params[ 'ENV' ][ 'cellName' ]

        git_pull = main.params[ 'GIT' ][ 'autoPull' ]
        checkout_branch = main.params[ 'GIT' ][ 'checkout' ]

        ONOS1_ip = main.params[ 'CTRL' ][ 'ip1' ]
        ONOS2_ip = main.params[ 'CTRL' ][ 'ip2' ]
        ONOS3_ip = main.params[ 'CTRL' ][ 'ip3' ]
        ONOS4_ip = main.params[ 'CTRL' ][ 'ip4' ]
        ONOS5_ip = main.params[ 'CTRL' ][ 'ip5' ]
        ONOS6_ip = main.params[ 'CTRL' ][ 'ip6' ]
        ONOS7_ip = main.params[ 'CTRL' ][ 'ip7' ]
        MN1_ip = main.params[ 'MN' ][ 'ip1' ]
        BENCH_ip = main.params[ 'BENCH' ][ 'ip' ]

        main.case( "Setting up test environment" )
        main.log.info( "copying topology event accumulator config file" +
                       " to ONOS package/etc/ directory" )
        topo_config_name = main.params[ 'TEST' ][ 'topo_config_name' ]
        topo_config =\
            main.params[ 'TEST' ][ 'topo_accumulator_config' ]
        main.ONOSbench.handle.sendline( "cp ~/" + topo_config +
                                        " ~/ONOS/tools/package/etc/" +
                                        topo_config_name )
        main.ONOSbench.handle.expect( "\$" )

        main.log.info( "Uninstalling previous instances" )
        #main.ONOSbench.onos_uninstall( node_ip=ONOS1_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS2_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS3_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS4_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS5_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS6_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS7_ip )

        main.log.report( "Setting up test environment" )

        main.step( "Creating cell file" )
        cell_file_result = main.ONOSbench.create_cell_file(
            BENCH_ip, cell_name, MN1_ip,
            "onos-core,onos-app-metrics",
            # ONOS1_ip, ONOS2_ip, ONOS3_ip )
            ONOS1_ip )

        main.step( "Applying cell file to environment" )
        cell_apply_result = main.ONOSbench.set_cell( cell_name )
        verify_cell_result = main.ONOSbench.verify_cell()

        main.step( "Removing raft logs" )
        main.ONOSbench.onos_remove_raft_logs()
        time.sleep( 10 )

        main.step( "Git checkout and pull " + checkout_branch )
        if git_pull == 'on':
            checkout_result = \
                main.ONOSbench.git_checkout( checkout_branch )
            pull_result = main.ONOSbench.git_pull()
        else:
            checkout_result = main.TRUE
            pull_result = main.TRUE
            main.log.info( "Skipped git checkout and pull" )

        main.log.report( "Commit information - " )
        main.ONOSbench.get_version()

        main.step( "Using mvn clean & install" )
        #mvn_result = main.ONOSbench.clean_install()
        mvn_result = main.TRUE

        main.step( "Set cell for ONOS cli env" )
        main.ONOS1cli.set_cell( cell_name )
        # main.ONOS2cli.set_cell( cell_name )
        # main.ONOS3cli.set_cell( cell_name )

        main.step( "Creating ONOS package" )
        package_result = main.ONOSbench.onos_package()

        # Start test with single node only
        main.step( "Installing ONOS package" )
        install1_result = main.ONOSbench.onos_install( node=ONOS1_ip )
        #install2_result = main.ONOSbench.onos_install( node=ONOS2_ip )
        #install3_result = main.ONOSbench.onos_install( node=ONOS3_ip )

        time.sleep( 10 )

        main.step( "Start onos cli" )
        cli1 = main.ONOS1cli.start_onos_cli( ONOS1_ip )
        #cli2 = main.ONOS2cli.start_onos_cli( ONOS2_ip )
        #cli3 = main.ONOS3cli.start_onos_cli( ONOS3_ip )

        main.step( "Enable metrics feature" )
        # main.ONOS1cli.feature_install( "onos-app-metrics" )

        utilities.assert_equals( expect=main.TRUE,
                                 actual=cell_file_result and cell_apply_result and
                                 verify_cell_result and checkout_result and
                                 pull_result and mvn_result and
                                 install1_result,  # and install2_result and
                                 # install3_result,
                                 onpass="Test Environment setup successful",
                                 onfail="Failed to setup test environment" )

    def CASE2( self, main ):
        """
        100 Switch discovery latency

        Important:
            This test case can be potentially dangerous if
            your machine has previously set iptables rules.
            One of the steps of the test case will flush
            all existing iptables rules.
        Note:
            You can specify the number of switches in the
            params file to adjust the switch discovery size
            ( and specify the corresponding topology in Mininet1
            .topo file )
        """
        import time
        import subprocess
        import os
        import requests
        import json
        import numpy

        ONOS_ip_list = []
        ONOS_ip_list.append( '0' )
        ONOS_ip_list.append( main.params[ 'CTRL' ][ 'ip1' ] )
        ONOS_ip_list.append( main.params[ 'CTRL' ][ 'ip2' ] )
        ONOS_ip_list.append( main.params[ 'CTRL' ][ 'ip3' ] )
        ONOS_ip_list.append( main.params[ 'CTRL' ][ 'ip4' ] )
        ONOS_ip_list.append( main.params[ 'CTRL' ][ 'ip5' ] )
        ONOS_ip_list.append( main.params[ 'CTRL' ][ 'ip6' ] )
        ONOS_ip_list.append( main.params[ 'CTRL' ][ 'ip7' ] )
        MN1_ip = main.params[ 'MN' ][ 'ip1' ]
        ONOS_user = main.params[ 'CTRL' ][ 'user' ]

        default_sw_port = main.params[ 'CTRL' ][ 'port1' ]

        # Number of iterations of case
        num_iter = main.params[ 'TEST' ][ 'numIter' ]
        iter_ignore = int( main.params[ 'TEST' ][ 'iterIgnore' ] )

        #***********
        # Global number of switches that change
        # throughout the test
        global num_sw
        global topo_iteration
        global cluster_count
        if topo_iteration == 1:
            num_sw = main.params[ 'TEST' ][ 'numSwitch1' ]
        elif topo_iteration == 2:
            num_sw = main.params[ 'TEST' ][ 'numSwitch2' ]
        elif topo_iteration == 3:
            num_sw = main.params[ 'TEST' ][ 'numSwitch3' ]
        elif topo_iteration == 4:
            num_sw = main.params[ 'TEST' ][ 'numSwitch4' ]
        #***********

        # Timestamp 'keys' for json metrics output.
        # These are subject to change, hence moved into params
        deviceTimestamp = main.params[ 'JSON' ][ 'deviceTimestamp' ]
        graphTimestamp = main.params[ 'JSON' ][ 'graphTimestamp' ]

        # Threshold for this test case
        sw_disc_threshold_str = main.params[ 'TEST' ][ 'swDisc100Threshold' ]
        sw_disc_threshold_obj = sw_disc_threshold_str.split( "," )
        sw_disc_threshold_min = int( sw_disc_threshold_obj[ 0 ] )
        sw_disc_threshold_max = int( sw_disc_threshold_obj[ 1 ] )

        assertion = main.TRUE
        sw_discovery_lat_list = []
        syn_ack_delta_list = []

        main.case( str( num_sw ) + " switches distributed across " +
                   str( cluster_count ) + " nodes convergence latency" )

        main.log.report( "Large topology convergence and scale-out test" )
        main.log.report( "Currently active ONOS node(s): " )
        report_str = "Node "
        for node in range( 1, cluster_count + 1 ):
            report_str += ( str( node ) + " " )
        main.log.report( report_str )
        main.log.report( "Topology size: " + str( num_sw ) + " switches" )

        main.step( "Distributing " + num_sw + " switches to each ONOS" )
        index = 1
        for node in range( 1, cluster_count + 1 ):
            for i in range( index, ( int( num_sw ) / cluster_count ) + index ):
                main.Mininet1.assign_sw_controller(
                    sw=str( i ),
                    ip1=ONOS_ip_list[ node ],
                    port1=default_sw_port )
            index = i + 1
        # for i in range( 1, int( num_sw )+1 ):
            # main.Mininet1.assign_sw_controller(
            # sw=str( i ),
            # ip1="10.128.174.1",
            #            port1="6633" )

        main.log.info( "Please check ptpd configuration to ensure " +
                       "all nodes' system times are in sync" )

        time.sleep( 10 )

        for i in range( 0, int( num_iter ) ):
            main.step( "Set iptables rule to block sw connections" )

            # INPUT rules
            main.ONOS1.handle.sendline(
                "sudo iptables -A INPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS2.handle.sendline(
                "sudo iptables -A INPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS3.handle.sendline(
                "sudo iptables -A INPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS4.handle.sendline(
                "sudo iptables -A INPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS5.handle.sendline(
                "sudo iptables -A INPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS6.handle.sendline(
                "sudo iptables -A INPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS7.handle.sendline(
                "sudo iptables -A INPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )

            # OUTPUT rules
            main.ONOS1.handle.sendline(
                "sudo iptables -A OUTPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS2.handle.sendline(
                "sudo iptables -A OUTPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS3.handle.sendline(
                "sudo iptables -A OUTPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS4.handle.sendline(
                "sudo iptables -A OUTPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS5.handle.sendline(
                "sudo iptables -A OUTPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS6.handle.sendline(
                "sudo iptables -A OUTPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )
            main.ONOS7.handle.sendline(
                "sudo iptables -A OUTPUT -p tcp -s " +
                MN1_ip + " --dport " + default_sw_port + " -j DROP" )

            main.log.info( "Please wait for switch connection to timeout" )

            # time.sleep( 60 )
            # if cluster_count >= 3:
            #    time.sleep( 60 )
            # if cluster_count >= 5:
            #    time.sleep( 30 )
            # if cluster_count >= 6:
            #    time.sleep( 30 )

            if cluster_count >= 3:
                main.ONOS1.handle.sendline(
                    "tshark -i eth0 -t e | " +
                    "grep 'SYN, ACK' | grep '6633' >" +
                    "/tmp/syn_ack_onos1_iter" +
                    str( i ) +
                    ".txt &" )
                main.ONOS2.handle.sendline(
                    "tshark -i eth0 -t e | " +
                    "grep 'SYN, ACK' | grep '6633' >" +
                    "/tmp/syn_ack_onos2_iter" +
                    str( i ) +
                    ".txt &" )
                main.ONOS3.handle.sendline(
                    "tshark -i eth0 -t e | " +
                    "grep 'SYN, ACK' | grep '6633' >" +
                    "/tmp/syn_ack_onos3_iter" +
                    str( i ) +
                    ".txt &" )
            if cluster_count >= 4:
                main.ONOS4.handle.sendline(
                    "tshark -i eth0 -t e | " +
                    "grep 'SYN, ACK' | grep '6633' >" +
                    "/tmp/syn_ack_onos4_iter" +
                    str( i ) +
                    ".txt &" )
            if cluster_count >= 5:
                main.ONOS5.handle.sendline(
                    "tshark -i eth0 -t e | " +
                    "grep 'SYN, ACK' | grep '6633' >" +
                    "/tmp/syn_ack_onos5_iter" +
                    str( i ) +
                    ".txt &" )
            if cluster_count >= 6:
                main.ONOS6.handle.sendline(
                    "tshark -i eth0 -t e | " +
                    "grep 'SYN, ACK' | grep '6633' >" +
                    "/tmp/syn_ack_onos6_iter" +
                    str( i ) +
                    ".txt &" )
            if cluster_count == 7:
                main.ONOS7.handle.sendline(
                    "tshark -i eth0 -t e | " +
                    "grep 'SYN, ACK' | grep '6633' >" +
                    "/tmp/syn_ack_onos6_iter" +
                    str( i ) +
                    ".txt &" )

            # NOTE:
            #       Delay before checking devices to
            #       help prevent timing out from CLI
            #       due to multiple command issuing
            time.sleep( 20 )

            loop = True
            loop_count = 0
            device_count = 0
            while loop_count < 60 and loop:
                main.log.info( "Checking devices for device down" )

                temp_len = 0
                device_str1 = main.ONOS1cli.devices(
                    node_ip=ONOS_ip_list[ 1 ] )
                device_json1 = json.loads( device_str1 )
                json_len = len( device_json1 )

                # NOTE: May want to check the rest of
                #      the ONOS instances for device down as well

                for device1 in device_json1:
                    temp_len = temp_len + 1
                    if device1[ 'available' ]:
                        loop = True
                        break
                    # if I'm on the last json object and I still haven't
                    # broken out of the loop, it means there were
                    # no available devices
                    elif temp_len == json_len - 1:
                        main.log.info( "Temp length: " + str( temp_len ) )
                        main.step( "Flushing iptables and obtaining t0" )
                        t0_system = time.time() * 1000

                        main.ONOS1.handle.sendline( "sudo iptables -F" )
                        main.ONOS2.handle.sendline( "sudo iptables -F" )
                        main.ONOS3.handle.sendline( "sudo iptables -F" )
                        main.ONOS4.handle.sendline( "sudo iptables -F" )
                        main.ONOS5.handle.sendline( "sudo iptables -F" )
                        main.ONOS6.handle.sendline( "sudo iptables -F" )
                        main.ONOS7.handle.sendline( "sudo iptables -F" )

                        loop = False
                        break

                loop_count += 1
                time.sleep( 1 )

            main.log.info( "System time t0: " + str( t0_system ) )

            counter_loop = 0
            counter_avail1 = 0
            counter_avail2 = 0
            counter_avail3 = 0
            counter_avail4 = 0
            counter_avail5 = 0
            counter_avail6 = 0
            counter_avail7 = 0
            onos1_dev = False
            onos2_dev = False
            onos3_dev = False
            onos4_dev = False
            onos5_dev = False
            onos6_dev = False
            onos7_dev = False

            # TODO: Think of a more elegant way to check all
            #      switches across all nodes
            # Goodluck debugging this loop
            while counter_loop < 60:
                for node in range( 1, cluster_count + 1 ):
                    if node == 1 and not onos1_dev:
                        main.log.info( "Checking node 1 for device " +
                                       "discovery" )
                        device_str_obj1 = main.ONOS1cli.devices(
                            node_ip=ONOS_ip_list[ 1 ] )
                        device_json1 = json.loads( device_str_obj1 )
                        for device1 in device_json1:
                            if device1[ 'available' ]:
                                counter_avail1 += 1
                                if counter_avail1 == int( num_sw ):
                                    onos1_dev = True
                                    main.log.info( "All devices have been" +
                                                   " discovered on ONOS1" )
                            else:
                                counter_avail1 = 0
                    if node == 2 and not onos2_dev:
                        main.log.info( "Checking node 2 for device " +
                                       "discovery" )
                        device_str_obj2 = main.ONOS2cli.devices(
                            node_ip=ONOS_ip_list[ 2 ] )
                        device_json2 = json.loads( device_str_obj2 )
                        for device2 in device_json2:
                            if device2[ 'available' ]:
                                counter_avail2 += 1
                                if counter_avail2 == int( num_sw ):
                                    onos2_dev = True
                                    main.log.info( "All devices have been" +
                                                   " discovered on ONOS2" )
                            else:
                                counter_avail2 = 0
                    if node == 3 and not onos3_dev:
                        main.log.info( "Checking node 3 for device " +
                                       "discovery" )
                        device_str_obj3 = main.ONOS3cli.devices(
                            node_ip=ONOS_ip_list[ 3 ] )
                        device_json3 = json.loads( device_str_obj3 )
                        for device3 in device_json3:
                            if device3[ 'available' ]:
                                counter_avail3 += 1
                                if counter_avail3 == int( num_sw ):
                                    onos3_dev = True
                                    main.log.info( "All devices have been" +
                                                   " discovered on ONOS3" )
                            else:
                                counter_avail3 = 0
                    if node == 4 and not onos4_dev:
                        main.log.info( "Checking node 4 for device " +
                                       "discovery" )
                        device_str_obj4 = main.ONOS4cli.devices(
                            node_ip=ONOS_ip_list[ 4 ] )
                        device_json4 = json.loads( device_str_obj4 )
                        for device4 in device_json4:
                            if device4[ 'available' ]:
                                counter_avail4 += 1
                                if counter_avail4 == int( num_sw ):
                                    onos4_dev = True
                                    main.log.info( "All devices have been" +
                                                   " discovered on ONOS4" )
                            else:
                                counter_avail4 = 0
                    if node == 5 and not onos5_dev:
                        main.log.info( "Checking node 5 for device " +
                                       "discovery" )
                        device_str_obj5 = main.ONOS5cli.devices(
                            node_ip=ONOS_ip_list[ 5 ] )
                        device_json5 = json.loads( device_str_obj5 )
                        for device5 in device_json5:
                            if device5[ 'available' ]:
                                counter_avail5 += 1
                                if counter_avail5 == int( num_sw ):
                                    onos5_dev = True
                                    main.log.info( "All devices have been" +
                                                   " discovered on ONOS5" )
                            else:
                                counter_avail5 = 0
                    if node == 6 and not onos6_dev:
                        main.log.info( "Checking node 6 for device " +
                                       "discovery" )
                        device_str_obj6 = main.ONOS6cli.devices(
                            node_ip=ONOS_ip_list[ 6 ] )
                        device_json6 = json.loads( device_str_obj6 )
                        for device6 in device_json6:
                            if device6[ 'available' ]:
                                counter_avail6 += 1
                                if counter_avail6 == int( num_sw ):
                                    onos6_dev = True
                                    main.log.info( "All devices have been" +
                                                   " discovered on ONOS6" )
                            else:
                                counter_avail6 = 0
                    if node == 7 and not onos7_dev:
                        main.log.info( "Checking node 7 for device " +
                                       "discovery" )
                        device_str_obj7 = main.ONOS7cli.devices(
                            node_ip=ONOS_ip_list[ 7 ] )
                        device_json7 = json.loads( device_str_obj7 )
                        for device7 in device_json7:
                            if device7[ 'available' ]:
                                counter_avail7 += 1
                                if counter_avail7 == int( num_sw ):
                                    onos7_dev = True
                                    main.log.info( "All devices have been" +
                                                   " discovered on ONOS7" )
                            else:
                                counter_avail7 = 0
                    # END node loop

                # TODO: clean up this mess of an if statements if possible
                # Treat each if as a separate test case with the given
                #     cluster count. Hence when the cluster count changes
                #     the desired calculations will be made
                if cluster_count == 1:
                    if onos1_dev:
                        main.log.info( "All devices have been discovered" +
                                       " on all ONOS instances" )
                        time.sleep( 5 )
                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_obj_1 = json.loads( json_str_metrics_1 )
                        graph_timestamp_1 =\
                            json_obj_1[ graphTimestamp ][ 'value' ]

                        graph_lat_1 = \
                            int( graph_timestamp_1 ) - int( t0_system )

                        main.log.info( "Graph Timestamp ONOS1: " +
                                       str( graph_timestamp_1 ) )

                        if graph_lat_1 > sw_disc_threshold_min\
                                and graph_lat_1 < sw_disc_threshold_max\
                                and int( i ) > iter_ignore:
                            sw_discovery_lat_list.append(
                                graph_lat_1 )
                            main.log.info(
                                "Sw discovery latency of " +
                                str( cluster_count ) +
                                " node(s): " +
                                str( graph_lat_1 ) +
                                " ms" )
                        else:
                            main.log.info( "Switch discovery latency " +
                                           "exceeded the threshold." )
                            main.log.info( str( graph_lat_1 ) + " ms" )
                        # Break while loop
                        break
                if cluster_count == 2:
                    if onos1_dev and onos2_dev:
                        main.log.info( "All devices have been discovered" +
                                       " on all " + str( cluster_count ) +
                                       " ONOS instances" )
                        time.sleep( 5 )

                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_obj_1 = json.loads( json_str_metrics_1 )
                        json_obj_2 = json.loads( json_str_metrics_2 )
                        graph_timestamp_1 =\
                            json_obj_1[ graphTimestamp ][ 'value' ]
                        graph_timestamp_2 =\
                            json_obj_2[ graphTimestamp ][ 'value' ]

                        graph_lat_1 = \
                            int( graph_timestamp_1 ) - int( t0_system )
                        graph_lat_2 = \
                            int( graph_timestamp_2 ) - int( t0_system )

                        main.log.info( "Graph Timestamp ONOS1: " +
                                       str( graph_timestamp_1 ) )
                        main.log.info( "Graph Timestamp ONOS2: " +
                                       str( graph_timestamp_2 ) )

                        max_graph_lat = max( graph_lat_1,
                                             graph_lat_2, graph_lat_3 )

                        if max_graph_lat > sw_disc_threshold_min\
                                and max_graph_lat < sw_disc_threshold_max\
                                and int( i ) > iter_ignore:
                            sw_discovery_lat_list.append(
                                max_graph_lat )
                            main.log.info(
                                "Sw discovery latency of " +
                                str( cluster_count ) +
                                " node(s): " +
                                str( max_graph_lat ) +
                                " ms" )
                        else:
                            main.log.info( "Switch discovery latency " +
                                           "exceeded the threshold." )
                            main.log.info( str( max_graph_lat ) + " ms" )
                        break
                if cluster_count == 3:
                    if onos1_dev and onos2_dev and onos3_dev:
                        main.log.info( "All devices have been discovered" +
                                       " on all " + str( cluster_count ) +
                                       " ONOS instances" )

                        # TODO: Investigate this sleep
                        #      added to 'pad' the results with
                        #      plenty of time to 'catch up'
                        time.sleep( 5 )

                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_obj_1 = json.loads( json_str_metrics_1 )
                        json_obj_2 = json.loads( json_str_metrics_2 )
                        json_obj_3 = json.loads( json_str_metrics_3 )
                        graph_timestamp_1 =\
                            json_obj_1[ graphTimestamp ][ 'value' ]
                        graph_timestamp_2 =\
                            json_obj_2[ graphTimestamp ][ 'value' ]
                        graph_timestamp_3 =\
                            json_obj_3[ graphTimestamp ][ 'value' ]

                        graph_lat_1 = \
                            int( graph_timestamp_1 ) - int( t0_system )
                        graph_lat_2 = \
                            int( graph_timestamp_2 ) - int( t0_system )
                        graph_lat_3 = \
                            int( graph_timestamp_3 ) - int( t0_system )

                        main.log.info( "Graph Timestamp ONOS1: " +
                                       str( graph_timestamp_1 ) )
                        main.log.info( "Graph Timestamp ONOS2: " +
                                       str( graph_timestamp_2 ) )
                        main.log.info( "Graph Timestamp ONOS3: " +
                                       str( graph_timestamp_3 ) )

                        max_graph_lat = max( graph_lat_1,
                                             graph_lat_2,
                                             graph_lat_3 )

                        if max_graph_lat > sw_disc_threshold_min\
                                and max_graph_lat < sw_disc_threshold_max\
                                and int( i ) > iter_ignore:
                            sw_discovery_lat_list.append(
                                max_graph_lat )
                            main.log.info(
                                "Sw discovery latency of " +
                                str( cluster_count ) +
                                " node(s): " +
                                str( max_graph_lat ) +
                                " ms" )
                        else:
                            main.log.info( "Switch discovery latency " +
                                           "exceeded the threshold." )
                            main.log.info( str( max_graph_lat ) + " ms" )

                        break
                if cluster_count == 4:
                    if onos1_dev and onos2_dev and onos3_dev and\
                       onos4_dev:
                        main.log.info( "All devices have been discovered" +
                                       " on all ONOS instances" )
                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_str_metrics_4 =\
                            main.ONOS4cli.topology_events_metrics()
                        json_obj_1 = json.loads( json_str_metrics_1 )
                        json_obj_2 = json.loads( json_str_metrics_2 )
                        json_obj_3 = json.loads( json_str_metrics_3 )
                        json_obj_4 = json.loads( json_str_metrics_4 )
                        graph_timestamp_1 =\
                            json_obj_1[ graphTimestamp ][ 'value' ]
                        graph_timestamp_2 =\
                            json_obj_2[ graphTimestamp ][ 'value' ]
                        graph_timestamp_3 =\
                            json_obj_3[ graphTimestamp ][ 'value' ]
                        graph_timestamp_4 =\
                            json_obj_4[ graphTimestamp ][ 'value' ]

                        graph_lat_1 = \
                            int( graph_timestamp_1 ) - int( t0_system )
                        graph_lat_2 = \
                            int( graph_timestamp_2 ) - int( t0_system )
                        graph_lat_3 = \
                            int( graph_timestamp_3 ) - int( t0_system )
                        graph_lat_4 = \
                            int( graph_timestamp_4 ) - int( t0_system )

                        main.log.info( "Graph Timestamp ONOS1: " +
                                       str( graph_timestamp_1 ) )
                        main.log.info( "Graph Timestamp ONOS2: " +
                                       str( graph_timestamp_2 ) )
                        main.log.info( "Graph Timestamp ONOS3: " +
                                       str( graph_timestamp_3 ) )
                        main.log.info( "Graph Timestamp ONOS4: " +
                                       str( graph_timestamp_4 ) )

                        max_graph_lat = max( graph_lat_1,
                                             graph_lat_2,
                                             graph_lat_3,
                                             graph_lat_4 )

                        if max_graph_lat > sw_disc_threshold_min\
                                and max_graph_lat < sw_disc_threshold_max\
                                and int( i ) > iter_ignore:
                            sw_discovery_lat_list.append(
                                max_graph_lat )
                            main.log.info(
                                "Sw discovery latency of " +
                                str( cluster_count ) +
                                " node(s): " +
                                str( max_graph_lat ) +
                                " ms" )
                        else:
                            main.log.info( "Switch discovery latency " +
                                           "exceeded the threshold." )
                            main.log.info( str( max_graph_lat ) + " ms" )

                        break
                if cluster_count == 5:
                    if onos1_dev and onos2_dev and onos3_dev and\
                       onos4_dev and onos5_dev:
                        main.log.info( "All devices have been discovered" +
                                       " on all ONOS instances" )

                        # TODO: Investigate this sleep
                        #      added to 'pad' the results with
                        #      plenty of time to 'catch up'
                        time.sleep( 5 )

                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_str_metrics_4 =\
                            main.ONOS4cli.topology_events_metrics()
                        json_str_metrics_5 =\
                            main.ONOS5cli.topology_events_metrics()
                        json_obj_1 = json.loads( json_str_metrics_1 )
                        json_obj_2 = json.loads( json_str_metrics_2 )
                        json_obj_3 = json.loads( json_str_metrics_3 )
                        json_obj_4 = json.loads( json_str_metrics_4 )
                        json_obj_5 = json.loads( json_str_metrics_5 )
                        graph_timestamp_1 =\
                            json_obj_1[ graphTimestamp ][ 'value' ]
                        graph_timestamp_2 =\
                            json_obj_2[ graphTimestamp ][ 'value' ]
                        graph_timestamp_3 =\
                            json_obj_3[ graphTimestamp ][ 'value' ]
                        graph_timestamp_4 =\
                            json_obj_4[ graphTimestamp ][ 'value' ]
                        graph_timestamp_5 =\
                            json_obj_5[ graphTimestamp ][ 'value' ]

                        graph_lat_1 = \
                            int( graph_timestamp_1 ) - int( t0_system )
                        graph_lat_2 = \
                            int( graph_timestamp_2 ) - int( t0_system )
                        graph_lat_3 = \
                            int( graph_timestamp_3 ) - int( t0_system )
                        graph_lat_4 = \
                            int( graph_timestamp_4 ) - int( t0_system )
                        graph_lat_5 = \
                            int( graph_timestamp_5 ) - int( t0_system )

                        main.log.info( "Graph Timestamp ONOS1: " +
                                       str( graph_timestamp_1 ) )
                        main.log.info( "Graph Timestamp ONOS2: " +
                                       str( graph_timestamp_2 ) )
                        main.log.info( "Graph Timestamp ONOS3: " +
                                       str( graph_timestamp_3 ) )
                        main.log.info( "Graph Timestamp ONOS4: " +
                                       str( graph_timestamp_4 ) )
                        main.log.info( "Graph Timestamp ONOS5: " +
                                       str( graph_timestamp_5 ) )

                        max_graph_lat = max( graph_lat_1,
                                             graph_lat_2,
                                             graph_lat_3,
                                             graph_lat_4,
                                             graph_lat_5 )

                        if max_graph_lat > sw_disc_threshold_min\
                                and max_graph_lat < sw_disc_threshold_max\
                                and int( i ) > iter_ignore:
                            sw_discovery_lat_list.append(
                                max_graph_lat )
                            main.log.info(
                                "Sw discovery latency of " +
                                str( cluster_count ) +
                                " node(s): " +
                                str( max_graph_lat ) +
                                " ms" )
                        else:
                            main.log.info( "Switch discovery latency " +
                                           "exceeded the threshold." )
                            main.log.info( str( max_graph_lat ) + " ms" )

                        break
                if cluster_count == 6:
                    if onos1_dev and onos2_dev and onos3_dev and\
                       onos4_dev and onos5_dev and onos6_dev:
                        main.log.info( "All devices have been discovered" +
                                       " on all ONOS instances" )
                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_str_metrics_4 =\
                            main.ONOS4cli.topology_events_metrics()
                        json_str_metrics_5 =\
                            main.ONOS5cli.topology_events_metrics()
                        json_str_metrics_6 =\
                            main.ONOS6cli.topology_events_metrics()
                        json_obj_1 = json.loads( json_str_metrics_1 )
                        json_obj_2 = json.loads( json_str_metrics_2 )
                        json_obj_3 = json.loads( json_str_metrics_3 )
                        json_obj_4 = json.loads( json_str_metrics_4 )
                        json_obj_5 = json.loads( json_str_metrics_5 )
                        json_obj_6 = json.loads( json_str_metrics_6 )
                        graph_timestamp_1 =\
                            json_obj_1[ graphTimestamp ][ 'value' ]
                        graph_timestamp_2 =\
                            json_obj_2[ graphTimestamp ][ 'value' ]
                        graph_timestamp_3 =\
                            json_obj_3[ graphTimestamp ][ 'value' ]
                        graph_timestamp_4 =\
                            json_obj_4[ graphTimestamp ][ 'value' ]
                        graph_timestamp_5 =\
                            json_obj_5[ graphTimestamp ][ 'value' ]
                        graph_timestamp_6 =\
                            json_obj_6[ graphTimestamp ][ 'value' ]

                        graph_lat_1 = \
                            int( graph_timestamp_1 ) - int( t0_system )
                        graph_lat_2 = \
                            int( graph_timestamp_2 ) - int( t0_system )
                        graph_lat_3 = \
                            int( graph_timestamp_3 ) - int( t0_system )
                        graph_lat_4 = \
                            int( graph_timestamp_4 ) - int( t0_system )
                        graph_lat_5 = \
                            int( graph_timestamp_5 ) - int( t0_system )
                        graph_lat_6 = \
                            int( graph_timestamp_6 ) - int( t0_system )

                        main.log.info( "Graph Timestamp ONOS1: " +
                                       str( graph_timestamp_1 ) )
                        main.log.info( "Graph Timestamp ONOS2: " +
                                       str( graph_timestamp_2 ) )
                        main.log.info( "Graph Timestamp ONOS3: " +
                                       str( graph_timestamp_3 ) )
                        main.log.info( "Graph Timestamp ONOS4: " +
                                       str( graph_timestamp_4 ) )
                        main.log.info( "Graph Timestamp ONOS5: " +
                                       str( graph_timestamp_5 ) )
                        main.log.info( "Graph Timestamp ONOS6: " +
                                       str( graph_timestamp_6 ) )

                        max_graph_lat = max( graph_lat_1,
                                             graph_lat_2,
                                             graph_lat_3,
                                             graph_lat_4,
                                             graph_lat_5,
                                             graph_lat_6 )

                        if max_graph_lat > sw_disc_threshold_min\
                                and max_graph_lat < sw_disc_threshold_max\
                                and int( i ) > iter_ignore:
                            sw_discovery_lat_list.append(
                                max_graph_lat )
                            main.log.info(
                                "Sw discovery latency of " +
                                str( cluster_count ) +
                                " node(s): " +
                                str( max_graph_lat ) +
                                " ms" )
                        else:
                            main.log.info( "Switch discovery latency " +
                                           "exceeded the threshold." )
                            main.log.info( str( max_graph_lat ) + " ms" )

                        break
                if cluster_count == 7:
                    if onos1_dev and onos2_dev and onos3_dev and\
                       onos4_dev and onos5_dev and onos6_dev and\
                       onos7_dev:
                        main.log.info( "All devices have been discovered" +
                                       " on all ONOS instances" )

                        # TODO: Investigate this sleep
                        #      added to 'pad' the results with
                        #      plenty of time to 'catch up'
                        time.sleep( 5 )

                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_str_metrics_4 =\
                            main.ONOS4cli.topology_events_metrics()
                        json_str_metrics_5 =\
                            main.ONOS5cli.topology_events_metrics()
                        json_str_metrics_6 =\
                            main.ONOS6cli.topology_events_metrics()
                        json_str_metrics_7 =\
                            main.ONOS7cli.topology_events_metrics()
                        json_obj_1 = json.loads( json_str_metrics_1 )
                        json_obj_2 = json.loads( json_str_metrics_2 )
                        json_obj_3 = json.loads( json_str_metrics_3 )
                        json_obj_4 = json.loads( json_str_metrics_4 )
                        json_obj_5 = json.loads( json_str_metrics_5 )
                        json_obj_6 = json.loads( json_str_metrics_6 )
                        json_obj_7 = json.loads( json_str_metrics_7 )
                        graph_timestamp_1 =\
                            json_obj_1[ graphTimestamp ][ 'value' ]
                        graph_timestamp_2 =\
                            json_obj_2[ graphTimestamp ][ 'value' ]
                        graph_timestamp_3 =\
                            json_obj_3[ graphTimestamp ][ 'value' ]
                        graph_timestamp_4 =\
                            json_obj_4[ graphTimestamp ][ 'value' ]
                        graph_timestamp_5 =\
                            json_obj_5[ graphTimestamp ][ 'value' ]
                        graph_timestamp_6 =\
                            json_obj_6[ graphTimestamp ][ 'value' ]
                        graph_timestamp_7 =\
                            json_obj_7[ graphTimestamp ][ 'value' ]

                        graph_lat_1 = \
                            int( graph_timestamp_1 ) - int( t0_system )
                        graph_lat_2 = \
                            int( graph_timestamp_2 ) - int( t0_system )
                        graph_lat_3 = \
                            int( graph_timestamp_3 ) - int( t0_system )
                        graph_lat_4 = \
                            int( graph_timestamp_4 ) - int( t0_system )
                        graph_lat_5 = \
                            int( graph_timestamp_5 ) - int( t0_system )
                        graph_lat_6 = \
                            int( graph_timestamp_6 ) - int( t0_system )
                        graph_lat_7 = \
                            int( graph_timestamp_7 ) - int( t0_system )

                        main.log.info( "Graph Timestamp ONOS1: " +
                                       str( graph_timestamp_1 ) )
                        main.log.info( "Graph Timestamp ONOS2: " +
                                       str( graph_timestamp_2 ) )
                        main.log.info( "Graph Timestamp ONOS3: " +
                                       str( graph_timestamp_3 ) )
                        main.log.info( "Graph Timestamp ONOS4: " +
                                       str( graph_timestamp_4 ) )
                        main.log.info( "Graph Timestamp ONOS5: " +
                                       str( graph_timestamp_5 ) )
                        main.log.info( "Graph Timestamp ONOS6: " +
                                       str( graph_timestamp_6 ) )
                        main.log.info( "Graph Timestamp ONOS7: " +
                                       str( graph_timestamp_7 ) )

                        max_graph_lat = max( graph_lat_1,
                                             graph_lat_2,
                                             graph_lat_3,
                                             graph_lat_4,
                                             graph_lat_5,
                                             graph_lat_6,
                                             graph_lat_7 )

                        if max_graph_lat > sw_disc_threshold_min\
                                and max_graph_lat < sw_disc_threshold_max\
                                and int( i ) > iter_ignore:
                            sw_discovery_lat_list.append(
                                max_graph_lat )
                            main.log.info(
                                "Sw discovery latency of " +
                                str( cluster_count ) +
                                " node(s): " +
                                str( max_graph_lat ) +
                                " ms" )
                        else:
                            main.log.info( "Switch discovery latency " +
                                           "exceeded the threshold." )
                            main.log.info( str( max_graph_lat ) + " ms" )

                        break

                counter_loop += 1
                time.sleep( 3 )
                # END WHILE LOOP

            # Below is used for reporting SYN / ACK timing
            # of all switches
            main.ONOS1.tshark_stop()
            syn_ack_timestamp_list = []
            if cluster_count < 3:
                # TODO: capture synack on nodes less than 3
                syn_ack_timestamp_list.append( 0 )

            if cluster_count >= 3:
                main.ONOS2.tshark_stop()
                main.ONOS3.tshark_stop()
                time.sleep( 5 )
                os.system(
                    "scp " +
                    ONOS_user +
                    "@" +
                    ONOS1_ip +
                    ":" +
                    "/tmp/syn_ack_onos1_iter" +
                    str( i ) +
                    ".txt /tmp/" )
                os.system(
                    "scp " +
                    ONOS_user +
                    "@" +
                    ONOS2_ip +
                    ":" +
                    "/tmp/syn_ack_onos2_iter" +
                    str( i ) +
                    ".txt /tmp/" )
                os.system(
                    "scp " +
                    ONOS_user +
                    "@" +
                    ONOS3_ip +
                    ":" +
                    "/tmp/syn_ack_onos3_iter" +
                    str( i ) +
                    ".txt /tmp/" )
                time.sleep( 5 )
                # Read each of the files and append all
                # SYN / ACK timestamps to the list
                with open( "/tmp/syn_ack_onos1_iter" + str( i ) + ".txt" ) as\
                        f_onos1:
                    for line in f_onos1:
                        line = line.split( " " )
                        try:
                            float( line[ 1 ] )
                            syn_ack_timestamp_list.append( line[ 1 ] )
                        except ValueError:
                            main.log.info( "String cannot be converted" )
                with open( "/tmp/syn_ack_onos2_iter" + str( i ) + ".txt" ) as\
                        f_onos2:
                    for line in f_onos2:
                        line = line.split( " " )
                        try:
                            float( line[ 1 ] )
                            syn_ack_timestamp_list.append( line[ 1 ] )
                        except ValueError:
                            main.log.info( "String cannot be converted" )
                with open( "/tmp/syn_ack_onos3_iter" + str( i ) + ".txt" ) as\
                        f_onos3:
                    for line in f_onos3:
                        line = line.split( " " )
                        try:
                            float( line[ 1 ] )
                            syn_ack_timestamp_list.append( line[ 1 ] )
                        except ValueError:
                            main.log.info( "String cannot be converted" )
            if cluster_count >= 4:
                main.ONOS4.tshark_stop()
                time.sleep( 5 )
                os.system(
                    "scp " +
                    ONOS_user +
                    "@" +
                    ONOS4_ip +
                    ":" +
                    "/tmp/syn_ack_onos4_iter" +
                    str( i ) +
                    ".txt /tmp/" )
                time.sleep( 5 )
                with open( "/tmp/syn_ack_onos4_iter" + str( i ) + ".txt" ) as\
                        f_onos4:
                    for line in f_onos4:
                        line = line.split( " " )
                        try:
                            float( line[ 1 ] )
                            syn_ack_timestamp_list.append( line[ 1 ] )
                        except ValueError:
                            main.log.info( "String cannot be converted" )
            if cluster_count >= 5:
                main.ONOS5.tshark_stop()
                time.sleep( 5 )
                os.system(
                    "scp " +
                    ONOS_user +
                    "@" +
                    ONOS5_ip +
                    ":" +
                    "/tmp/syn_ack_onos5_iter" +
                    str( i ) +
                    ".txt /tmp/" )
                time.sleep( 5 )
                with open( "/tmp/syn_ack_onos5_iter" + str( i ) + ".txt" ) as\
                        f_onos5:
                    for line in f_onos5:
                        line = line.split( " " )
                        try:
                            float( line[ 1 ] )
                            syn_ack_timestamp_list.append( line[ 1 ] )
                        except ValueError:
                            main.log.info( "String cannot be converted" )
            if cluster_count >= 6:
                main.ONOS6.tshark_stop()
                time.sleep( 5 )
                os.system(
                    "scp " +
                    ONOS_user +
                    "@" +
                    ONOS6_ip +
                    ":" +
                    "/tmp/syn_ack_onos6_iter" +
                    str( i ) +
                    ".txt /tmp/" )
                time.sleep( 5 )
                with open( "/tmp/syn_ack_onos6_iter" + str( i ) + ".txt" ) as\
                        f_onos6:
                    for line in f_onos6:
                        line = line.split( " " )
                        try:
                            float( line[ 1 ] )
                            syn_ack_timestamp_list.append( line[ 1 ] )
                        except ValueError:
                            main.log.info( "String cannot be converted" )
            if cluster_count == 7:
                main.ONOS7.tshark_stop()
                time.sleep( 5 )
                os.system(
                    "scp " +
                    ONOS_user +
                    "@" +
                    ONOS7_ip +
                    ":" +
                    "/tmp/syn_ack_onos7_iter" +
                    str( i ) +
                    ".txt /tmp/" )
                time.sleep( 5 )
                with open( "/tmp/syn_ack_onos7_iter" + str( i ) + ".txt" ) as\
                        f_onos7:
                    for line in f_onos7:
                        line = line.split( " " )
                        try:
                            float( line[ 1 ] )
                            syn_ack_timestamp_list.append( line[ 1 ] )
                        except ValueError:
                            main.log.info( "String cannot be converted" )

            # Sort the list by timestamp
            syn_ack_timestamp_list = sorted( syn_ack_timestamp_list )
            print "syn_ack_-1  " + str( syn_ack_timestamp_list )

            syn_ack_delta =\
                int( float( syn_ack_timestamp_list[ -1 ] ) * 1000 ) -\
                int( float( syn_ack_timestamp_list[ 0 ] ) * 1000 )

            main.log.info( "Switch connection attempt delta iteration " +
                           str( i ) + ": " + str( syn_ack_delta ) )
            syn_ack_delta_list.append( syn_ack_delta )
            # END ITERATION LOOP
        # REPORT HERE

        if len( sw_discovery_lat_list ) > 0:
            sw_lat_avg = sum( sw_discovery_lat_list ) / \
                len( sw_discovery_lat_list )
            sw_lat_dev = numpy.std( sw_discovery_lat_list )
        else:
            sw_lat_avg = 0
            sw_lat_dev = 0
            assertion = main.FALSE

        main.log.report( "Switch connection attempt time avg " +
                         "(last sw SYN/ACK time - first sw SYN/ACK time) " +
                         str( sum( syn_ack_delta_list ) /
                              len( syn_ack_delta_list ) ) +
                         " ms" )
        main.log.report( str( num_sw ) + " Switch discovery lat for " +
                         str( cluster_count ) + " instance(s): " )
        main.log.report( "Avg: " +
                         str( sw_lat_avg ) +
                         " ms  " +
                         "Std Deviation: " +
                         str( round( sw_lat_dev, 1 ) ) +
                         " ms" )

        utilities.assert_equals(
            expect=main.TRUE,
            actual=assertion,
            onpass="Switch discovery convergence latency" +
            " for " +
            str( cluster_count ) +
            " nodes successful",
            onfail="Switch discovery convergence latency" +
            " test failed" )

    def CASE3( self, main ):
        """
        Increase number of nodes and initiate CLI
        """
        import time
        import subprocess
        import os
        import requests
        import json

        ONOS1_ip = main.params[ 'CTRL' ][ 'ip1' ]
        ONOS2_ip = main.params[ 'CTRL' ][ 'ip2' ]
        ONOS3_ip = main.params[ 'CTRL' ][ 'ip3' ]
        ONOS4_ip = main.params[ 'CTRL' ][ 'ip4' ]
        ONOS5_ip = main.params[ 'CTRL' ][ 'ip5' ]
        ONOS6_ip = main.params[ 'CTRL' ][ 'ip6' ]
        ONOS7_ip = main.params[ 'CTRL' ][ 'ip7' ]

        cell_name = main.params[ 'ENV' ][ 'cellName' ]

        MN1_ip = main.params[ 'MN' ][ 'ip1' ]
        BENCH_ip = main.params[ 'BENCH' ][ 'ip' ]

        # NOTE:We start with cluster_count at 3. The first
        # case already initialized ONOS1. Increase the
        # cluster count and start from 3.
        # You can optionally change the increment to
        # test steps of node sizes, such as 3,5,7

        global cluster_count
        cluster_count += 2
        main.log.report( "Increasing cluster size to " +
                         str( cluster_count ) )

        install_result = main.FALSE
        # Supports up to 7 node configuration
        # TODO: Cleanup this ridiculous repetitive code
        if cluster_count == 3:
            install_result = \
                main.ONOSbench.onos_install( node=ONOS2_ip )
            install_result = \
                main.ONOSbench.onos_install( node=ONOS3_ip )
            time.sleep( 5 )
            main.log.info( "Starting CLI" )
            main.ONOS2cli.start_onos_cli( ONOS2_ip )
            main.ONOS3cli.start_onos_cli( ONOS3_ip )
            main.ONOS1cli.add_node( ONOS2_ip, ONOS2_ip )
            main.ONOS1cli.add_node( ONOS3_ip, ONOS3_ip )

        if cluster_count == 4:
            main.log.info( "Installing ONOS on node 4" )
            install_result = \
                main.ONOSbench.onos_install( node=ONOS4_ip )
            time.sleep( 5 )
            main.log.info( "Starting CLI" )
            main.ONOS4cli.start_onos_cli( ONOS4_ip )
            main.ONOS1cli.add_node( ONOS4_ip, ONOS4_ip )

        elif cluster_count == 5:
            main.log.info( "Installing ONOS on nodes 4 and 5" )
            install_result2 = \
                main.ONOSbench.onos_install( options="", node=ONOS4_ip )
            install_result3 = \
                main.ONOSbench.onos_install( options="", node=ONOS5_ip )
            time.sleep( 5 )
            main.log.info( "Starting CLI" )
            main.ONOS4cli.start_onos_cli( ONOS4_ip )
            main.ONOS5cli.start_onos_cli( ONOS5_ip )
            main.ONOS1cli.add_node( ONOS4_ip, ONOS4_ip )
            main.ONOS1cli.add_node( ONOS5_ip, ONOS5_ip )
            install_result = install_result2 and install_result3

        elif cluster_count == 6:
            main.log.info( "Installing ONOS on nodes 4, 5,and 6" )
            install_result1 = \
                main.ONOSbench.onos_install( options="", node=ONOS4_ip )
            install_result2 = \
                main.ONOSbench.onos_install( options="", node=ONOS5_ip )
            install_result3 = \
                main.ONOSbench.onos_install( node=ONOS6_ip )
            time.sleep( 5 )
            main.log.info( "Starting CLI" )
            main.ONOS4cli.start_onos_cli( ONOS4_ip )
            main.ONOS5cli.start_onos_cli( ONOS5_ip )
            main.ONOS6cli.start_onos_cli( ONOS6_ip )
            main.ONOS1cli.add_node( ONOS4_ip, ONOS4_ip )
            main.ONOS1cli.add_node( ONOS5_ip, ONOS5_ip )
            main.ONOS1cli.add_node( ONOS6_ip, ONOS6_ip )
            install_result = install_result1 and install_result2 and\
                install_result3

        elif cluster_count == 7:
            main.log.info( "Installing ONOS on nodes 4, 5, 6,and 7" )
            install_result3 = \
                main.ONOSbench.onos_install( node=ONOS6_ip )
            install_result4 = \
                main.ONOSbench.onos_install( node=ONOS7_ip )
            main.log.info( "Starting CLI" )
            main.ONOS4cli.start_onos_cli( ONOS4_ip )
            main.ONOS5cli.start_onos_cli( ONOS5_ip )
            main.ONOS6cli.start_onos_cli( ONOS6_ip )
            main.ONOS7cli.start_onos_cli( ONOS7_ip )
            main.ONOS1cli.add_node( ONOS4_ip, ONOS4_ip )
            main.ONOS1cli.add_node( ONOS5_ip, ONOS5_ip )
            main.ONOS1cli.add_node( ONOS6_ip, ONOS6_ip )
            main.ONOS1cli.add_node( ONOS7_ip, ONOS7_ip )

            install_result = \
                install_result3 and install_result4

        time.sleep( 5 )

        if install_result == main.TRUE:
            assertion = main.TRUE
        else:
            assertion = main.FALSE

        utilities.assert_equals(
            expect=main.TRUE,
            actual=assertion,
            onpass="Scale out to " +
            str( cluster_count ) +
            " nodes successful",
            onfail="Scale out to " +
            str( cluster_count ) +
            " nodes failed" )

    def CASE4( self, main ):
        """
        Cleanup ONOS nodes and Increase topology size
        """
        # TODO: use meaningful assertion
        assertion = main.TRUE

        ONOS1_ip = main.params[ 'CTRL' ][ 'ip1' ]
        ONOS2_ip = main.params[ 'CTRL' ][ 'ip2' ]
        ONOS3_ip = main.params[ 'CTRL' ][ 'ip3' ]
        ONOS4_ip = main.params[ 'CTRL' ][ 'ip4' ]
        ONOS5_ip = main.params[ 'CTRL' ][ 'ip5' ]
        ONOS6_ip = main.params[ 'CTRL' ][ 'ip6' ]
        ONOS7_ip = main.params[ 'CTRL' ][ 'ip7' ]
        MN1_ip = main.params[ 'MN' ][ 'ip1' ]
        BENCH_ip = main.params[ 'BENCH' ][ 'ip' ]

        main.log.info( "Uninstalling previous instances" )
        main.ONOSbench.onos_uninstall( node_ip=ONOS2_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS3_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS4_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS5_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS6_ip )
        main.ONOSbench.onos_uninstall( node_ip=ONOS7_ip )

        global topo_iteration
        global cluster_count
        cluster_count = 1
        topo_iteration += 1

        main.log.report( "Increasing topology size" )
        utilities.assert_equals( expect=main.TRUE, actual=assertion,
                                 onpass="Topology size increased successfully",
                                 onfail="Topology size was not increased" )