TestON/tests/TopoConvNext/TopoConvNext.py

#TopoPerfNext
#
#Topology Convergence scale-out test for ONOS-next
#NOTE: This test supports up to 7 nodes scale-out scenario
#
#NOTE: Ensure that you have 'tablet.json' file 
#      in the onos/tools/package/config directory
#NOTE: You must start this test initially with 3 nodes
#
#andrew@onlab.us

import time
import sys
import os
import re

class TopoConvNext:
    def __init__(self):
        self.default = ''

    def CASE1(self, main):
        '''
        ONOS startup sequence
        '''
        import time

        #******
        #Global cluster count for scale-out purposes
        global cluster_count 
        global topo_iteration
        topo_iteration = 1
        cluster_count = 3 
        #******
        cell_name = main.params['ENV']['cellName']

        git_pull = main.params['GIT']['autoPull']
        checkout_branch = main.params['GIT']['checkout']

        ONOS1_ip = main.params['CTRL']['ip1']
        ONOS2_ip = main.params['CTRL']['ip2']
        ONOS3_ip = main.params['CTRL']['ip3']
        ONOS4_ip = main.params['CTRL']['ip4']
        ONOS5_ip = main.params['CTRL']['ip5']
        ONOS6_ip = main.params['CTRL']['ip6']
        ONOS7_ip = main.params['CTRL']['ip7']
        MN1_ip = main.params['MN']['ip1']
        BENCH_ip = main.params['BENCH']['ip']

        main.case("Setting up test environment")
        main.log.info("copying topology event accumulator config file"+\
                " to ONOS package/etc/ directory")
        topo_config_name = main.params['TEST']['topo_config_name']
        topo_config =\
                main.params['TEST']['topo_accumulator_config']
        main.ONOSbench.handle.sendline("cp ~/"+topo_config+\
            " ~/ONOS/tools/package/etc/"+\
            topo_config_name)
        main.ONOSbench.handle.expect("\$")

        main.log.info("Uninstalling previous instances")
        #main.ONOSbench.onos_uninstall(node_ip = ONOS1_ip)
        #main.ONOSbench.onos_uninstall(node_ip = ONOS2_ip)
        #main.ONOSbench.onos_uninstall(node_ip = ONOS3_ip)
        main.ONOSbench.onos_uninstall(node_ip = ONOS4_ip)
        main.ONOSbench.onos_uninstall(node_ip = ONOS5_ip)
        main.ONOSbench.onos_uninstall(node_ip = ONOS6_ip)
        main.ONOSbench.onos_uninstall(node_ip = ONOS7_ip)
      
        main.log.report("Setting up test environment")

        main.step("Creating cell file")
        cell_file_result = main.ONOSbench.create_cell_file(
                BENCH_ip, cell_name, MN1_ip, 
                "onos-core,onos-app-metrics", 
                        ONOS1_ip, ONOS2_ip, ONOS3_ip)
                #ONOS1_ip)
                    
        main.step("Applying cell file to environment")
        cell_apply_result = main.ONOSbench.set_cell(cell_name)
        verify_cell_result = main.ONOSbench.verify_cell()
        
        main.step("Removing raft logs")
        main.ONOSbench.onos_remove_raft_logs()
        
        main.step("Git checkout and pull "+checkout_branch)
        if git_pull == 'on':
            checkout_result = \
                    main.ONOSbench.git_checkout(checkout_branch)
            pull_result = main.ONOSbench.git_pull()
        else:
            checkout_result = main.TRUE
            pull_result = main.TRUE
            main.log.info("Skipped git checkout and pull")

        main.log.report("Commit information - ")
        main.ONOSbench.get_version()

        main.step("Using mvn clean & install")
        #mvn_result = main.ONOSbench.clean_install()
        mvn_result = main.TRUE

        main.step("Set cell for ONOS cli env")
        main.ONOS1cli.set_cell(cell_name)
        main.ONOS2cli.set_cell(cell_name)
        main.ONOS3cli.set_cell(cell_name)
    
        main.step("Creating ONOS package")
        package_result = main.ONOSbench.onos_package()

        #Start test with single node only
        main.step("Installing ONOS package")
        install1_result = main.ONOSbench.onos_install(node=ONOS1_ip)
        install2_result = main.ONOSbench.onos_install(node=ONOS2_ip)
        install3_result = main.ONOSbench.onos_install(node=ONOS3_ip)

        time.sleep(10)

        main.step("Start onos cli")
        cli1 = main.ONOS1cli.start_onos_cli(ONOS1_ip)
        cli2 = main.ONOS2cli.start_onos_cli(ONOS2_ip)
        cli3 = main.ONOS3cli.start_onos_cli(ONOS3_ip)

        main.step("Enable metrics feature")
        #main.ONOS1cli.feature_install("onos-app-metrics")

        utilities.assert_equals(expect=main.TRUE,
                actual= cell_file_result and cell_apply_result and\
                        verify_cell_result and checkout_result and\
                        pull_result and mvn_result and\
                        install1_result and install2_result and\
                        install3_result,
                onpass="Test Environment setup successful",
                onfail="Failed to setup test environment")
    
    def CASE2(self, main):
        '''
        100 Switch discovery latency

        Important:
            This test case can be potentially dangerous if 
            your machine has previously set iptables rules.
            One of the steps of the test case will flush
            all existing iptables rules.
        Note:
            You can specify the number of switches in the 
            params file to adjust the switch discovery size
            (and specify the corresponding topology in Mininet1 
            .topo file)
        '''
        import time
        import subprocess
        import os
        import requests
        import json
        import numpy

        ONOS_ip_list = []
        ONOS_ip_list.append('0')
        ONOS_ip_list.append(main.params['CTRL']['ip1'])
        ONOS_ip_list.append(main.params['CTRL']['ip2'])
        ONOS_ip_list.append(main.params['CTRL']['ip3'])
        ONOS_ip_list.append(main.params['CTRL']['ip4'])
        ONOS_ip_list.append(main.params['CTRL']['ip5'])
        ONOS_ip_list.append(main.params['CTRL']['ip6'])
        ONOS_ip_list.append(main.params['CTRL']['ip7'])
        MN1_ip = main.params['MN']['ip1']
        ONOS_user = main.params['CTRL']['user']

        default_sw_port = main.params['CTRL']['port1']
       
        #Number of iterations of case
        num_iter = main.params['TEST']['numIter']
        iter_ignore = int(main.params['TEST']['iterIgnore'])

        #***********
        #Global number of switches that change 
        #throughout the test
        global num_sw
        global topo_iteration    
        if topo_iteration == 1:
            num_sw = main.params['TEST']['numSwitch1']
        elif topo_iteration == 2:
            num_sw = main.params['TEST']['numSwitch2']
        elif topo_iteration == 3:
            num_sw = main.params['TEST']['numSwitch3']
        #***********

        #Timestamp 'keys' for json metrics output.
        #These are subject to change, hence moved into params
        deviceTimestamp = main.params['JSON']['deviceTimestamp']
        graphTimestamp = main.params['JSON']['graphTimestamp']
        
        #Threshold for this test case
        sw_disc_threshold_str = main.params['TEST']['swDisc100Threshold']
        sw_disc_threshold_obj = sw_disc_threshold_str.split(",")
        sw_disc_threshold_min = int(sw_disc_threshold_obj[0])
        sw_disc_threshold_max = int(sw_disc_threshold_obj[1])

        assertion = main.TRUE
        sw_discovery_lat_list = []
        syn_ack_delta_list = []

        main.case(str(num_sw)+" switches distributed across "+
                str(cluster_count)+" nodes convergence latency")
       
        main.log.report("Large topology convergence and scale-out test")
        main.log.report("Currently active ONOS node(s): ")
        report_str = "Node "
        for node in range(1, cluster_count+1):
            report_str += (str(node) + " ") 
        main.log.report(report_str)
        main.log.report("Topology size: "+str(num_sw)+" switches")

        main.step("Distributing "+num_sw+" switches to each ONOS")
        index = 1 
        for node in range(1, cluster_count+1):
            for i in range(index, (int(num_sw)/cluster_count)+index):
                main.Mininet1.assign_sw_controller(
                        sw=str(i),
                        ip1=ONOS_ip_list[node],
                        port1=default_sw_port)
            index = i+1 

        main.log.info("Please check ptpd configuration to ensure "+\
                "all nodes' system times are in sync")

        time.sleep(10)

        for i in range(0, int(num_iter)):
            main.step("Set iptables rule to block sw connections")
               
            #INPUT rules
            main.ONOS1.handle.sendline(
                    "sudo iptables -A INPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS2.handle.sendline(
                    "sudo iptables -A INPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS3.handle.sendline(
                    "sudo iptables -A INPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS4.handle.sendline(
                    "sudo iptables -A INPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS5.handle.sendline(
                    "sudo iptables -A INPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS6.handle.sendline(
                    "sudo iptables -A INPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS7.handle.sendline(
                    "sudo iptables -A INPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
               
            #OUTPUT rules
            main.ONOS1.handle.sendline(
                    "sudo iptables -A OUTPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS2.handle.sendline(
                    "sudo iptables -A OUTPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS3.handle.sendline(
                    "sudo iptables -A OUTPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS4.handle.sendline(
                    "sudo iptables -A OUTPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS5.handle.sendline(
                    "sudo iptables -A OUTPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS6.handle.sendline(
                    "sudo iptables -A OUTPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")
            main.ONOS7.handle.sendline(
                    "sudo iptables -A OUTPUT -p tcp -s "+
                    MN1_ip+" --dport "+default_sw_port+" -j DROP")

            main.log.info("Please wait for switch connection to timeout")
           

            #time.sleep(60)
            #if cluster_count >= 3:
            #    time.sleep(60)
            #if cluster_count >= 5:
            #    time.sleep(30)
            #if cluster_count >= 6:
            #    time.sleep(30)

            if cluster_count >= 3:
                main.ONOS1.tshark_grep("SYN, ACK",
                        "/tmp/syn_ack_onos1_iter"+str(i)+".txt")
                main.ONOS2.tshark_grep("SYN, ACK",
                        "/tmp/syn_ack_onos2_iter"+str(i)+".txt")
                main.ONOS3.tshark_grep("SYN, ACK",
                        "/tmp/syn_ack_onos3_iter"+str(i)+".txt")
            if cluster_count >= 4:
                main.ONOS4.tshark_grep("SYN, ACK",
                        "/tmp/syn_ack_onos4_iter"+str(i)+".txt")
            if cluster_count >= 5:
                main.ONOS5.tshark_grep("SYN, ACK",
                        "/tmp/syn_ack_onos5_iter"+str(i)+".txt")
            if cluster_count >= 6:
                main.ONOS6.tshark_grep("SYN, ACK",
                        "/tmp/syn_ack_onos6_iter"+str(i)+".txt")
            if cluster_count == 7:
                main.ONOS7.tshark_grep("SYN, ACK",
                        "/tmp/syn_ack_onos7_iter"+str(i)+".txt")
           
            loop_count = 0
            device_count1 = 0
            device_count2 = 0
            device_count3 = 0
            while loop_count < 60: 
                main.log.info("Checking devices for device down")
                device_str1 = main.ONOS1cli.devices(
                        node_ip=ONOS_ip_list[1])
                device_json1 = json.loads(device_str1)
                
                #TODO: Modular check of all nodes for 
                #      device down discovery

                #device_str2 = main.ONOS2cli.devices(
                        #        node_ip=ONOS_ip_list[2])
                #device_json2 = json.loads(device_str2)
                #device_str3 = main.ONOS3cli.devices(
                #        node_ip=ONOS_ip_list[3])
                #device_json3 = json.loads(device_str3)
                for device1 in device_json1:
                    if device1['available'] == False:
                        device_count1 += 1
                    else:
                        device_count1 = 0
                #for device2 in device_json2:
                #    if device2['available'] == False:
                #        print device_count2
                #        device_count2 += 1
                #    else:
                #        device_count2 = 0
                #for device3 in device_json3:
                #    if device3['available'] == False:
                #        print device_count3
                #        device_count3 += 1
                #    else:
                #        device_count3 = 0
                
                if device_count1\
                        >= int(num_sw)*int(cluster_count):
                    main.step("Flushing iptables and obtaining t0")
                    t0_system = time.time()*1000
                    main.ONOS1.handle.sendline("sudo iptables -F")
                    main.ONOS2.handle.sendline("sudo iptables -F")
                    main.ONOS3.handle.sendline("sudo iptables -F")
                    main.ONOS4.handle.sendline("sudo iptables -F")
                    main.ONOS5.handle.sendline("sudo iptables -F")
                    main.ONOS6.handle.sendline("sudo iptables -F")
                    main.ONOS7.handle.sendline("sudo iptables -F")
                    
                    break
                
                time.sleep(1)

            main.log.info("System time t0: "+str(t0_system))

            counter_loop = 0
            counter_avail1 = 0
            counter_avail2 = 0
            counter_avail3 = 0
            counter_avail4 = 0
            counter_avail5 = 0
            counter_avail6 = 0
            counter_avail7 = 0
            onos1_dev = False
            onos2_dev = False
            onos3_dev = False
            onos4_dev = False
            onos5_dev = False
            onos6_dev = False
            onos7_dev = False

            #TODO: Think of a more elegant way to check all 
            #      switches across all nodes
            #Goodluck debugging this loop
            while counter_loop < 60:
                for node in range(1, cluster_count+1):
                    if node == 1 and not onos1_dev:
                        main.log.info("Checking node 1 for device "+
                            "discovery")
                        device_str_obj1 = main.ONOS1cli.devices(
                            node_ip=ONOS_ip_list[1])
                        device_json1 = json.loads(device_str_obj1)
                        for device1 in device_json1:
                            if device1['available'] == True:
                                counter_avail1 += 1
                                if counter_avail1 == int(num_sw):
                                    onos1_dev = True
                                    main.log.info("All devices have been"+
                                            " discovered on ONOS1")
                            else:
                                counter_avail1 = 0
                    if node == 2 and not onos2_dev:
                        main.log.info("Checking node 2 for device "+
                            "discovery")
                        device_str_obj2 = main.ONOS2cli.devices(
                            node_ip=ONOS_ip_list[2])
                        device_json2 = json.loads(device_str_obj2)
                        for device2 in device_json2:
                            if device2['available'] == True:
                                counter_avail2 += 1
                                if counter_avail2 == int(num_sw):
                                    onos2_dev = True
                                    main.log.info("All devices have been"+
                                            " discovered on ONOS2")
                            else:
                                counter_avail2 = 0
                    if node == 3 and not onos3_dev:
                        main.log.info("Checking node 3 for device "+
                            "discovery")
                        device_str_obj3 = main.ONOS3cli.devices(
                            node_ip=ONOS_ip_list[3])
                        device_json3 = json.loads(device_str_obj3)
                        for device3 in device_json3:
                            if device3['available'] == True:
                                counter_avail3 += 1
                                if counter_avail3 == int(num_sw):
                                    onos3_dev = True
                                    main.log.info("All devices have been"+
                                            " discovered on ONOS3")
                            else:
                                counter_avail3 = 0
                    if node == 4 and not onos4_dev:
                        main.log.info("Checking node 4 for device "+
                            "discovery")
                        device_str_obj4 = main.ONOS4cli.devices(
                            node_ip=ONOS_ip_list[4])
                        device_json4 = json.loads(device_str_obj4)
                        for device4 in device_json4:
                            if device4['available'] == True:
                                counter_avail4 += 1
                                if counter_avail4 == int(num_sw):
                                    onos4_dev = True
                                    main.log.info("All devices have been"+
                                            " discovered on ONOS4")
                            else:
                                counter_avail4 = 0
                    if node == 5 and not onos5_dev:
                        main.log.info("Checking node 5 for device "+
                            "discovery")
                        device_str_obj5 = main.ONOS5cli.devices(
                            node_ip=ONOS_ip_list[5])
                        device_json5 = json.loads(device_str_obj5)
                        for device5 in device_json5:
                            if device5['available'] == True:
                                counter_avail5 += 1
                                if counter_avail5 == int(num_sw):
                                    onos5_dev = True
                                    main.log.info("All devices have been"+
                                            " discovered on ONOS5")
                            else:
                                counter_avail5 = 0
                    if node == 6 and not onos6_dev:
                        main.log.info("Checking node 6 for device "+
                            "discovery")
                        device_str_obj6 = main.ONOS6cli.devices(
                            node_ip=ONOS_ip_list[6])
                        device_json6 = json.loads(device_str_obj6)
                        for device6 in device_json6:
                            if device6['available'] == True:
                                counter_avail6 += 1
                                if counter_avail6 == int(num_sw):
                                    onos6_dev = True
                                    main.log.info("All devices have been"+
                                            " discovered on ONOS6")
                            else:
                                counter_avail6 = 0
                    if node == 7 and not onos7_dev:
                        main.log.info("Checking node 7 for device "+
                            "discovery")
                        device_str_obj7 = main.ONOS7cli.devices(
                            node_ip=ONOS_ip_list[7])
                        device_json7 = json.loads(device_str_obj7)
                        for device7 in device_json7:
                            if device7['available'] == True:
                                counter_avail7 += 1
                                if counter_avail7 == int(num_sw):
                                    onos7_dev = True
                                    main.log.info("All devices have been"+
                                            " discovered on ONOS7")
                            else:
                                counter_avail7 = 0
                    #END node loop
              
                #TODO: clean up this mess of an if statements if possible
                #Treat each if as a separate test case with the given
                #     cluster count. Hence when the cluster count changes
                #     the desired calculations will be made
                if cluster_count == 1:
                    if onos1_dev:
                        main.log.info("All devices have been discovered"+
                            " on all ONOS instances")
                        time.sleep(5)
                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_obj_1 = json.loads(json_str_metrics_1)
                        graph_timestamp_1 =\
                            json_obj_1[graphTimestamp]['value']
                        
                        graph_lat_1 = \
                            int(graph_timestamp_1) - int(t0_system)
                        
                        if graph_lat_1 > sw_disc_threshold_min\
                            and graph_lat_1 < sw_disc_threshold_max\
                            and int(i) > iter_ignore:
                            sw_discovery_lat_list.append(
                                    graph_lat_1)
                            main.log.info("Sw discovery latency of "+
                                str(cluster_count)+" node(s): "+
                                str(graph_lat_1)+" ms")
                        else:
                            main.log.info("Switch discovery latency "+
                                "exceeded the threshold.")
                            main.log.info(graph_lat_1)
                        #Break while loop 
                        break
                if cluster_count == 2:
                    if onos1_dev and onos2_dev:
                        main.log.info("All devices have been discovered"+
                            " on all "+str(cluster_count)+
                            " ONOS instances")
                        time.sleep(5)

                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_obj_1 = json.loads(json_str_metrics_1)
                        json_obj_2 = json.loads(json_str_metrics_2)
                        graph_timestamp_1 =\
                            json_obj_1[graphTimestamp]['value']
                        graph_timestamp_2 =\
                            json_obj_2[graphTimestamp]['value']
                        
                        graph_lat_1 = \
                            int(graph_timestamp_1) - int(t0_system)
                        graph_lat_2 = \
                            int(graph_timestamp_2) - int(t0_system)

                        avg_graph_lat = \
                            (int(graph_lat_1) +\
                             int(graph_lat_2)) / 2

                        if avg_graph_lat > sw_disc_threshold_min\
                            and avg_graph_lat < sw_disc_threshold_max\
                            and int(i) > iter_ignore:
                            sw_discovery_lat_list.append(
                                    avg_graph_lat)
                            main.log.info("Sw discovery latency of "+
                                str(cluster_count)+" node(s): "+
                                str(avg_graph_lat)+" ms")
                        else:
                            main.log.info("Switch discovery latency "+
                                "exceeded the threshold.")
                            main.log.info(avg_graph_lat)
                        break
                if cluster_count == 3:
                    if onos1_dev and onos2_dev and onos3_dev:
                        main.log.info("All devices have been discovered"+
                            " on all "+str(cluster_count)+
                            " ONOS instances")
                        
                        #TODO: Investigate this sleep
                        #      added to 'pad' the results with 
                        #      plenty of time to 'catch up'
                        time.sleep(5)

                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_obj_1 = json.loads(json_str_metrics_1)
                        json_obj_2 = json.loads(json_str_metrics_2)
                        json_obj_3 = json.loads(json_str_metrics_3)
                        graph_timestamp_1 =\
                            json_obj_1[graphTimestamp]['value']
                        graph_timestamp_2 =\
                            json_obj_2[graphTimestamp]['value']
                        graph_timestamp_3 =\
                            json_obj_3[graphTimestamp]['value']
                        
                        graph_lat_1 = \
                            int(graph_timestamp_1) - int(t0_system)
                        graph_lat_2 = \
                            int(graph_timestamp_2) - int(t0_system)
                        graph_lat_3 = \
                            int(graph_timestamp_3) - int(t0_system)

                        main.log.info("DEBUG: graph_timestamp_1: "+
                                str(graph_timestamp_1))

                        avg_graph_lat = \
                            (int(graph_lat_1) +\
                             int(graph_lat_2) +\
                             int(graph_lat_3)) / 3 
                        
                        if avg_graph_lat > sw_disc_threshold_min\
                            and avg_graph_lat < sw_disc_threshold_max\
                            and int(i) > iter_ignore:
                            sw_discovery_lat_list.append(
                                    avg_graph_lat)
                            main.log.info("Sw discovery latency of "+
                                str(cluster_count)+" node(s): "+
                                str(avg_graph_lat)+" ms")
                        else:
                            main.log.info("Switch discovery latency "+
                                "exceeded the threshold.")
                            main.log.info(avg_graph_lat)
                        
                        break
                if cluster_count == 4:
                    if onos1_dev and onos2_dev and onos3_dev and\
                       onos4_dev:
                        main.log.info("All devices have been discovered"+
                            " on all ONOS instances")
                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_str_metrics_4 =\
                            main.ONOS4cli.topology_events_metrics()
                        json_obj_1 = json.loads(json_str_metrics_1)
                        json_obj_2 = json.loads(json_str_metrics_2)
                        json_obj_3 = json.loads(json_str_metrics_3)
                        json_obj_4 = json.loads(json_str_metrics_4)
                        graph_timestamp_1 =\
                            json_obj_1[graphTimestamp]['value']
                        graph_timestamp_2 =\
                            json_obj_2[graphTimestamp]['value']
                        graph_timestamp_3 =\
                            json_obj_3[graphTimestamp]['value']
                        graph_timestamp_4 =\
                            json_obj_4[graphTimestamp]['value']
                        
                        graph_lat_1 = \
                            int(graph_timestamp_1) - int(t0_system)
                        graph_lat_2 = \
                            int(graph_timestamp_2) - int(t0_system)
                        graph_lat_3 = \
                            int(graph_timestamp_3) - int(t0_system)
                        graph_lat_4 = \
                            int(graph_timestamp_4) - int(t0_system)

                        avg_graph_lat = \
                            (int(graph_lat_1) +\
                             int(graph_lat_2) +\
                             int(graph_lat_3) +\
                             int(graph_lat_4)) / 4 
                        
                        if avg_graph_lat > sw_disc_threshold_min\
                            and avg_graph_lat < sw_disc_threshold_max\
                            and int(i) > iter_ignore:
                            sw_discovery_lat_list.append(
                                    avg_graph_lat)
                            main.log.info("Sw discovery latency of "+
                                str(cluster_count)+" node(s): "+
                                str(avg_graph_lat)+" ms")
                        else:
                            main.log.info("Switch discovery latency "+
                                "exceeded the threshold.")
                            main.log.info(avg_graph_lat)
                
                        break
                if cluster_count == 5:
                    if onos1_dev and onos2_dev and onos3_dev and\
                       onos4_dev and onos5_dev:
                        main.log.info("All devices have been discovered"+
                            " on all ONOS instances")
                        
                        #TODO: Investigate this sleep
                        #      added to 'pad' the results with 
                        #      plenty of time to 'catch up'
                        time.sleep(5)
                        
                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_str_metrics_4 =\
                            main.ONOS4cli.topology_events_metrics()
                        json_str_metrics_5 =\
                            main.ONOS5cli.topology_events_metrics()
                        json_obj_1 = json.loads(json_str_metrics_1)
                        json_obj_2 = json.loads(json_str_metrics_2)
                        json_obj_3 = json.loads(json_str_metrics_3)
                        json_obj_4 = json.loads(json_str_metrics_4)
                        json_obj_5 = json.loads(json_str_metrics_5)
                        graph_timestamp_1 =\
                            json_obj_1[graphTimestamp]['value']
                        graph_timestamp_2 =\
                            json_obj_2[graphTimestamp]['value']
                        graph_timestamp_3 =\
                            json_obj_3[graphTimestamp]['value']
                        graph_timestamp_4 =\
                            json_obj_4[graphTimestamp]['value']
                        graph_timestamp_5 =\
                            json_obj_5[graphTimestamp]['value']
                        
                        graph_lat_1 = \
                            int(graph_timestamp_1) - int(t0_system)
                        graph_lat_2 = \
                            int(graph_timestamp_2) - int(t0_system)
                        graph_lat_3 = \
                            int(graph_timestamp_3) - int(t0_system)
                        graph_lat_4 = \
                            int(graph_timestamp_4) - int(t0_system)
                        graph_lat_5 = \
                            int(graph_timestamp_5) - int(t0_system)

                        avg_graph_lat = \
                            (int(graph_lat_1) +\
                             int(graph_lat_2) +\
                             int(graph_lat_3) +\
                             int(graph_lat_4) +\
                             int(graph_lat_5)) / 5 
                        
                        if avg_graph_lat > sw_disc_threshold_min\
                            and avg_graph_lat < sw_disc_threshold_max\
                            and int(i) > iter_ignore:
                            sw_discovery_lat_list.append(
                                    avg_graph_lat)
                            main.log.info("Sw discovery latency of "+
                                str(cluster_count)+" node(s): "+
                                str(avg_graph_lat)+" ms")
                        else:
                            main.log.info("Switch discovery latency "+
                                "exceeded the threshold.")
                            main.log.info(avg_graph_lat)
                
                        break
                if cluster_count == 6:
                    if onos1_dev and onos2_dev and onos3_dev and\
                       onos4_dev and onos5_dev and onos6_dev:
                        main.log.info("All devices have been discovered"+
                            " on all ONOS instances")
                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_str_metrics_4 =\
                            main.ONOS4cli.topology_events_metrics()
                        json_str_metrics_5 =\
                            main.ONOS5cli.topology_events_metrics()
                        json_str_metrics_6 =\
                            main.ONOS6cli.topology_events_metrics()
                        json_obj_1 = json.loads(json_str_metrics_1)
                        json_obj_2 = json.loads(json_str_metrics_2)
                        json_obj_3 = json.loads(json_str_metrics_3)
                        json_obj_4 = json.loads(json_str_metrics_4)
                        json_obj_5 = json.loads(json_str_metrics_5)
                        json_obj_6 = json.loads(json_str_metrics_6)
                        graph_timestamp_1 =\
                            json_obj_1[graphTimestamp]['value']
                        graph_timestamp_2 =\
                            json_obj_2[graphTimestamp]['value']
                        graph_timestamp_3 =\
                            json_obj_3[graphTimestamp]['value']
                        graph_timestamp_4 =\
                            json_obj_4[graphTimestamp]['value']
                        graph_timestamp_5 =\
                            json_obj_5[graphTimestamp]['value']
                        graph_timestamp_6 =\
                            json_obj_6[graphTimestamp]['value']
                        
                        graph_lat_1 = \
                            int(graph_timestamp_1) - int(t0_system)
                        graph_lat_2 = \
                            int(graph_timestamp_2) - int(t0_system)
                        graph_lat_3 = \
                            int(graph_timestamp_3) - int(t0_system)
                        graph_lat_4 = \
                            int(graph_timestamp_4) - int(t0_system)
                        graph_lat_5 = \
                            int(graph_timestamp_5) - int(t0_system)
                        graph_lat_6 = \
                            int(graph_timestamp_6) - int(t0_system)

                        avg_graph_lat = \
                            (int(graph_lat_1) +\
                             int(graph_lat_2) +\
                             int(graph_lat_3) +\
                             int(graph_lat_4) +\
                             int(graph_lat_5) +\
                             int(graph_lat_6)) / 6 
                        
                        if avg_graph_lat > sw_disc_threshold_min\
                            and avg_graph_lat < sw_disc_threshold_max\
                            and int(i) > iter_ignore:
                            sw_discovery_lat_list.append(
                                    avg_graph_lat)
                            main.log.info("Sw discovery latency of "+
                                str(cluster_count)+" node(s): "+
                                str(avg_graph_lat)+" ms")
                        else:
                            main.log.info("Switch discovery latency "+
                                "exceeded the threshold.")
                            main.log.info(avg_graph_lat)
                        
                        break
                if cluster_count == 7:
                    if onos1_dev and onos2_dev and onos3_dev and\
                       onos4_dev and onos5_dev and onos6_dev and\
                       onos7_dev:
                        main.log.info("All devices have been discovered"+
                            " on all ONOS instances")
                        
                        #TODO: Investigate this sleep
                        #      added to 'pad' the results with 
                        #      plenty of time to 'catch up'
                        time.sleep(5)
                        
                        json_str_metrics_1 =\
                            main.ONOS1cli.topology_events_metrics()
                        json_str_metrics_2 =\
                            main.ONOS2cli.topology_events_metrics()
                        json_str_metrics_3 =\
                            main.ONOS3cli.topology_events_metrics()
                        json_str_metrics_4 =\
                            main.ONOS4cli.topology_events_metrics()
                        json_str_metrics_5 =\
                            main.ONOS5cli.topology_events_metrics()
                        json_str_metrics_6 =\
                            main.ONOS6cli.topology_events_metrics()
                        json_str_metrics_7 =\
                            main.ONOS7cli.topology_events_metrics()
                        json_obj_1 = json.loads(json_str_metrics_1)
                        json_obj_2 = json.loads(json_str_metrics_2)
                        json_obj_3 = json.loads(json_str_metrics_3)
                        json_obj_4 = json.loads(json_str_metrics_4)
                        json_obj_5 = json.loads(json_str_metrics_5)
                        json_obj_6 = json.loads(json_str_metrics_6)
                        json_obj_7 = json.loads(json_str_metrics_7)
                        graph_timestamp_1 =\
                            json_obj_1[graphTimestamp]['value']
                        graph_timestamp_2 =\
                            json_obj_2[graphTimestamp]['value']
                        graph_timestamp_3 =\
                            json_obj_3[graphTimestamp]['value']
                        graph_timestamp_4 =\
                            json_obj_4[graphTimestamp]['value']
                        graph_timestamp_5 =\
                            json_obj_5[graphTimestamp]['value']
                        graph_timestamp_6 =\
                            json_obj_6[graphTimestamp]['value']
                        graph_timestamp_7 =\
                            json_obj_7[graphTimestamp]['value']
                        
                        graph_lat_1 = \
                            int(graph_timestamp_1) - int(t0_system)
                        graph_lat_2 = \
                            int(graph_timestamp_2) - int(t0_system)
                        graph_lat_3 = \
                            int(graph_timestamp_3) - int(t0_system)
                        graph_lat_4 = \
                            int(graph_timestamp_4) - int(t0_system)
                        graph_lat_5 = \
                            int(graph_timestamp_5) - int(t0_system)
                        graph_lat_6 = \
                            int(graph_timestamp_6) - int(t0_system)
                        graph_lat_7 = \
                            int(graph_timestamp_7) - int(t0_system)

                        avg_graph_lat = \
                            (int(graph_lat_1) +\
                             int(graph_lat_2) +\
                             int(graph_lat_3) +\
                             int(graph_lat_4) +\
                             int(graph_lat_5) +\
                             int(graph_lat_6) +\
                             int(graph_lat_7)) / 7 
                        
                        if avg_graph_lat > sw_disc_threshold_min\
                            and avg_graph_lat < sw_disc_threshold_max\
                            and int(i) > iter_ignore:
                            sw_discovery_lat_list.append(
                                    avg_graph_lat)
                            main.log.info("Sw discovery latency of "+
                                str(cluster_count)+" node(s): "+
                                str(avg_graph_lat)+" ms")
                        else:
                            main.log.info("Switch discovery latency "+
                                "exceeded the threshold.")
                            main.log.info(avg_graph_lat)
                        
                        break
                
                counter_loop += 1
                time.sleep(3)
                #END WHILE LOOP            
            
            main.ONOS1.tshark_stop()
            syn_ack_timestamp_list = []

            if cluster_count >= 3:
                main.ONOS2.tshark_stop() 
                main.ONOS3.tshark_stop()
                time.sleep(5)
                os.system("scp "+ONOS_user+"@"+ONOS1_ip+":"+
                    "/tmp/syn_ack_onos1_iter"+str(i)+".txt /tmp/")
                os.system("scp "+ONOS_user+"@"+ONOS2_ip+":"+
                    "/tmp/syn_ack_onos2_iter"+str(i)+".txt /tmp/")
                os.system("scp "+ONOS_user+"@"+ONOS3_ip+":"+
                    "/tmp/syn_ack_onos3_iter"+str(i)+".txt /tmp/")
                time.sleep(5)
                #Read each of the files and append all
                #SYN / ACK timestamps to the list
                with open("/tmp/syn_ack_onos1_iter"+str(i)+".txt") as\
                     f_onos1:
                    for line in f_onos1:
                        line = line.split(" ")
                        syn_ack_timestamp_list.append(line[1])
                with open("/tmp/syn_ack_onos2_iter"+str(i)+".txt") as\
                     f_onos2:
                    for line in f_onos2:
                        line = line.split(" ")
                        syn_ack_timestamp_list.append(line[1])
                with open("/tmp/syn_ack_onos3_iter"+str(i)+".txt") as\
                     f_onos3:
                    for line in f_onos3:
                        line = line.split(" ")
                        syn_ack_timestamp_list.append(line[1])
            if cluster_count >= 4:
                main.ONOS4.tshark_stop() 
                time.sleep(5)
                os.system("scp "+ONOS_user+"@"+ONOS4_ip+":"+
                    "/tmp/syn_ack_onos4_iter"+str(i)+".txt /tmp/")
                time.sleep(5)
                with open("/tmp/syn_ack_onos4_iter"+str(i)+".txt") as\
                     f_onos4:
                    for line in f_onos4:
                        line = line.split(" ")
                        syn_ack_timestamp_list.append(line[1])
            if cluster_count >= 5:
                main.ONOS5.tshark_stop()
                time.sleep(5)
                os.system("scp "+ONOS_user+"@"+ONOS5_ip+":"+
                    "/tmp/syn_ack_onos5_iter"+str(i)+".txt /tmp/")
                time.sleep(5)
                with open("/tmp/syn_ack_onos5_iter"+str(i)+".txt") as\
                     f_onos5:
                    for line in f_onos5:
                        line = line.split(" ")
                        syn_ack_timestamp_list.append(line[1])
            if cluster_count >= 6:
                main.ONOS6.tshark_stop()
                time.sleep(5)
                os.system("scp "+ONOS_user+"@"+ONOS6_ip+":"+
                    "/tmp/syn_ack_onos6_iter"+str(i)+".txt /tmp/")
                time.sleep(5)
                with open("/tmp/syn_ack_onos6_iter"+str(i)+".txt") as\
                     f_onos6:
                    for line in f_onos6:
                        line = line.split(" ")
                        syn_ack_timestamp_list.append(line[1])
            if cluster_count == 7:
                main.ONOS7.tshark_stop()
                time.sleep(5)
                os.system("scp "+ONOS_user+"@"+ONOS7_ip+":"+
                    "/tmp/syn_ack_onos7_iter"+str(i)+".txt /tmp/")
                time.sleep(5)
                with open("/tmp/syn_ack_onos7_iter"+str(i)+".txt") as\
                     f_onos7:
                    for line in f_onos7:
                        line = line.split(" ")
                        syn_ack_timestamp_list.append(line[1])
          
            #Sort the list by timestamp
            syn_ack_timestamp_list = sorted(syn_ack_timestamp_list)
            
            syn_ack_delta =\
                    int(float(syn_ack_timestamp_list[-1])*1000) -\
                    int(float(syn_ack_timestamp_list[0])*1000) 

            main.log.info("Switch connection attempt delta iteration "+
                    str(i)+": "+str(syn_ack_delta))
            syn_ack_delta_list.append(syn_ack_delta)
            #END ITERATION LOOP
        #REPORT HERE 

        if len(sw_discovery_lat_list) > 0:
            sw_lat_avg = sum(sw_discovery_lat_list) / \
                     len(sw_discovery_lat_list)
            sw_lat_dev = numpy.std(sw_discovery_lat_list)
        else: 
            assertion = main.FALSE
        
        main.log.report("Switch connection attempt time avg "+
            "(last sw SYN/ACK time - first sw SYN/ACK time) "+
            str(sum(syn_ack_delta_list)/len(syn_ack_delta_list)) +
            " ms")
        main.log.report(str(num_sw)+" Switch discovery lat for "+\
            str(cluster_count)+" instance(s): ")
        main.log.report("Avg: "+str(sw_lat_avg)+" ms  "+
            "Std Deviation: "+str(round(sw_lat_dev,1))+" ms")

        utilities.assert_equals(expect=main.TRUE, actual=assertion,
                onpass="Switch discovery convergence latency" +\
                        " for "+str(cluster_count)+" nodes successful",
                onfail="Switch discovery convergence latency" +\
                        " test failed")
        
    def CASE3(self, main):
        '''
        Increase number of nodes and initiate CLI
        '''
        import time
        import subprocess
        import os
        import requests
        import json
       
        ONOS1_ip = main.params['CTRL']['ip1']
        ONOS2_ip = main.params['CTRL']['ip2']
        ONOS3_ip = main.params['CTRL']['ip3']
        ONOS4_ip = main.params['CTRL']['ip4']
        ONOS5_ip = main.params['CTRL']['ip5']
        ONOS6_ip = main.params['CTRL']['ip6']
        ONOS7_ip = main.params['CTRL']['ip7']
        
        cell_name = main.params['ENV']['cellName']
        
        MN1_ip = main.params['MN']['ip1']
        BENCH_ip = main.params['BENCH']['ip']

        #NOTE:We start with cluster_count at 3. The first 
        #case already initialized ONOS1. Increase the
        #cluster count and start from 3.
        #You can optionally change the increment to
        #test steps of node sizes, such as 3,5,7
       
        global cluster_count
        cluster_count += 2 
        main.log.report("Increasing cluster size to "+
            str(cluster_count))

        install_result = main.FALSE
        #Supports up to 7 node configuration
        #TODO: Cleanup this ridiculous repetitive code 
        if cluster_count == 3:
            install_result = \
                main.ONOSbench.onos_install(node=ONOS3_ip)
            time.sleep(5)
            main.log.info("Starting CLI")
            main.ONOS3cli.start_onos_cli(ONOS3_ip)
            main.ONOS1cli.add_node(ONOS3_ip, ONOS3_ip)
            
        if cluster_count == 4:
            main.log.info("Installing ONOS on node 4")
            install_result = \
                main.ONOSbench.onos_install(node=ONOS4_ip)
            time.sleep(5)
            main.log.info("Starting CLI")
            main.ONOS4cli.start_onos_cli(ONOS4_ip)
            main.ONOS1cli.add_node(ONOS4_ip, ONOS4_ip)
        
        elif cluster_count == 5:
            main.log.info("Installing ONOS on nodes 4 and 5")
            install_result1 = \
                main.ONOSbench.onos_install(node=ONOS3_ip)
            install_result2 = \
                main.ONOSbench.onos_install(node=ONOS4_ip)
            install_result3 = \
                main.ONOSbench.onos_install(node=ONOS5_ip)
            time.sleep(5)
            main.log.info("Starting CLI")
            main.ONOS3cli.start_onos_cli(ONOS3_ip)
            main.ONOS4cli.start_onos_cli(ONOS4_ip)
            main.ONOS5cli.start_onos_cli(ONOS5_ip)
            main.ONOS1cli.add_node(ONOS3_ip, ONOS3_ip)
            main.ONOS1cli.add_node(ONOS4_ip, ONOS4_ip)
            main.ONOS1cli.add_node(ONOS5_ip, ONOS5_ip)
            install_result = install_result1 and install_result2

        elif cluster_count == 6:
            main.log.info("Installing ONOS on nodes 4, 5,and 6")
            install_result1 = \
                main.ONOSbench.onos_install(options="",node=ONOS4_ip)
            install_result2 = \
                main.ONOSbench.onos_install(options="",node=ONOS5_ip)
            install_result3 = \
                main.ONOSbench.onos_install(node=ONOS6_ip)
            time.sleep(5)
            main.log.info("Starting CLI")
            main.ONOS4cli.start_onos_cli(ONOS4_ip)
            main.ONOS5cli.start_onos_cli(ONOS5_ip)
            main.ONOS6cli.start_onos_cli(ONOS6_ip)
            main.ONOS1cli.add_node(ONOS4_ip, ONOS4_ip)
            main.ONOS1cli.add_node(ONOS5_ip, ONOS5_ip)
            main.ONOS1cli.add_node(ONOS6_ip, ONOS6_ip)
            install_result = install_result1 and install_result2 and\
                    install_result3

        elif cluster_count == 7:
            main.log.info("Installing ONOS on nodes 4, 5, 6,and 7")
            install_result1 = \
                main.ONOSbench.onos_install(options="",node=ONOS4_ip)
            install_result2 = \
                main.ONOSbench.onos_install(options="",node=ONOS5_ip)
            install_result3 = \
                main.ONOSbench.onos_install(options="",node=ONOS6_ip)
            install_result4 = \
                main.ONOSbench.onos_install(node=ONOS7_ip)
            main.log.info("Starting CLI")
            main.ONOS4cli.start_onos_cli(ONOS4_ip)
            main.ONOS5cli.start_onos_cli(ONOS5_ip)
            main.ONOS6cli.start_onos_cli(ONOS6_ip)
            main.ONOS7cli.start_onos_cli(ONOS7_ip) 
            main.ONOS1cli.add_node(ONOS4_ip, ONOS4_ip)
            main.ONOS1cli.add_node(ONOS5_ip, ONOS5_ip)
            main.ONOS1cli.add_node(ONOS6_ip, ONOS6_ip)
            main.ONOS1cli.add_node(ONOS7_ip, ONOS7_ip)

            install_result = install_result1 and install_result2 and\
                    install_result3 and install_result4

        time.sleep(5)

        if install_result == main.TRUE:
            assertion = main.TRUE
        else:
            assertion = main.FALSE
        
        utilities.assert_equals(expect=main.TRUE, actual=assertion,
                onpass="Scale out to "+str(cluster_count)+\
                       " nodes successful",
                onfail="Scale out to "+str(cluster_count)+\
                       " nodes failed")


    def CASE4(self, main):
        '''
        Cleanup ONOS nodes and Increase topology size
        '''
        #TODO: use meaningful assertion
        assertion=main.TRUE

        ONOS1_ip = main.params['CTRL']['ip1']
        ONOS2_ip = main.params['CTRL']['ip2']
        ONOS3_ip = main.params['CTRL']['ip3']
        ONOS4_ip = main.params['CTRL']['ip4']
        ONOS5_ip = main.params['CTRL']['ip5']
        ONOS6_ip = main.params['CTRL']['ip6']
        ONOS7_ip = main.params['CTRL']['ip7']
        MN1_ip = main.params['MN']['ip1']
        BENCH_ip = main.params['BENCH']['ip']

        main.log.info("Uninstalling previous instances")
        main.ONOSbench.onos_uninstall(node_ip = ONOS2_ip)
        main.ONOSbench.onos_uninstall(node_ip = ONOS3_ip)
        main.ONOSbench.onos_uninstall(node_ip = ONOS4_ip)
        main.ONOSbench.onos_uninstall(node_ip = ONOS5_ip)
        main.ONOSbench.onos_uninstall(node_ip = ONOS6_ip)
        main.ONOSbench.onos_uninstall(node_ip = ONOS7_ip)
        
        global topo_iteration
        global cluster_count
        cluster_count = 3 
        #cluster_count = 1
        topo_iteration += 1

        main.log.report("Increasing topology size")
        utilities.assert_equals(expect=main.TRUE, actual=assertion,
            onpass="Topology size increased successfully",
            onfail="Topology size was not increased")