Merge branch 'ONOS-Next' of https://github.com/OPENNETWORKINGLAB/ONLabTest into ONOS-Next

Conflicts:
	TestON/drivers/common/cli/onosclidriver.py
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 4916ba5..2391d3a 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1193,8 +1193,8 @@
             main.cleanup()
             main.exit()
 
-    def push_test_intents(self, dpid_src, dpid_dst, num_intents,
-            report=True):
+    def push_test_intents(self, dpid_src, dpid_dst, num_intents, 
+            num_mult="", app_id="", report=True):
         '''
         Description:
             Push a number of intents in a batch format to 
@@ -1204,12 +1204,22 @@
             * dpid_dst: specify destination dpid
             * num_intents: specify number of intents to push
         Optional:
+            * num_mult: number multiplier for multiplying
+              the number of intents specified
+            * app_id: specify the application id init to further
+              modularize the intents
             * report: default True, returns latency information
         '''
         try:
             cmd = "push-test-intents "+\
                   str(dpid_src)+" "+str(dpid_dst)+" "+\
                   str(num_intents)
+            
+            if num_mult:
+                cmd += " " + str(num_mult)
+                if app_id:
+                    cmd += " " + str(app_id)
+            
             self.handle.sendline(cmd)
             self.handle.expect(cmd)
             self.handle.expect("onos>")
@@ -1221,8 +1231,20 @@
             handle = ansi_escape.sub('', handle)
     
             if report:
+                lat_result = []
                 main.log.info(handle)
-                return handle
+                #Split result by newline
+                newline = handle.split("\r\r\n")
+                #Ignore the first object of list, which is empty
+                newline = newline[1:]
+                #Some sloppy parsing method to get the latency
+                for result in newline:
+                    result = result.split(": ")
+                    #Append the first result of second parse
+                    lat_result.append(result[1].split(" ")[0])
+
+                print lat_result 
+                return lat_result 
             else:
                 return main.TRUE
 
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 991bdf3..6bb46a0 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -963,6 +963,52 @@
             main.cleanup()
             main.exit()
 
+    def push_test_intents_shell(self, dpid_src, dpid_dst, num_intents,
+            dir_file, onos_ip, num_mult="", app_id="", report=True):    
+        '''  
+        Description:
+            Use the linux prompt to push test intents to 
+            better parallelize the results than the CLI
+        Required:
+            * dpid_src: specify source dpid
+            * dpid_dst: specify destination dpid
+            * num_intents: specify number of intents to push
+            * dir_file: specify directory and file name to save
+              results
+            * onos_ip: specify the IP of ONOS to install on
+        NOTE: 
+            You must invoke this command at linux shell prompt
+        '''
+        try: 
+            #Create the string to sendline 
+            base_cmd = "onos "+str(onos_ip)+" push-test-intents "
+            add_dpid = base_cmd + str(dpid_src) + " " + str(dpid_dst)  
+            if not num_mult:
+                add_intents = add_dpid + " " + str(num_intents)
+            elif num_mult:
+                add_intents = add_dpid + " " + str(num_intents) + " " +\
+                              str(num_mult)
+                if app_id:
+                    add_app = add_intents + " " + str(app_id) 
+                else:
+                    add_app = add_intents
+
+            send_cmd = add_app + " > " + str(dir_file) + " &" 
+            main.log.info("Send cmd: "+send_cmd)
+
+            self.handle.sendline(send_cmd)
+
+        except pexpect.EOF:
+            main.log.error(self.name + ": EOF exception found")
+            main.log.error(self.name + ":    " + self.handle.before)
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.info(self.name+" ::::::")
+            main.log.error( traceback.print_exc())
+            main.log.info(self.name+" ::::::")
+            main.cleanup()
+            main.exit() 
 
     def get_topology(self,topology_output):
         '''
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 7bf7667..00d00ed 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -50,7 +50,7 @@
         if self.port:
             self.handle =pexpect.spawn('ssh -p '+self.port+' '+self.user_name+'@'+self.ip_address,maxread=50000)
         else :
-            self.handle =pexpect.spawn('ssh -X '+self.user_name+'@'+self.ip_address,maxread=50000)
+            self.handle =pexpect.spawn('ssh -X '+self.user_name+'@'+self.ip_address,maxread=1000000,timeout=60)
 
         self.handle.logfile = self.logfile_handler
 	i = 5
diff --git a/TestON/tests/IntentPerfNext/IntentPerfNext.params b/TestON/tests/IntentPerfNext/IntentPerfNext.params
index 87e7998..168a067 100644
--- a/TestON/tests/IntentPerfNext/IntentPerfNext.params
+++ b/TestON/tests/IntentPerfNext/IntentPerfNext.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>1,4</testcases>
+    <testcases>1,4,5,4,5,4,5,4</testcases>
 
     <ENV>
         <cellName>intent_perf_test</cellName>
@@ -19,6 +19,10 @@
         <port2>6633</port2>
         <ip3>10.128.174.3</ip3>
         <port3>6633</port3>
+        <ip4>10.128.174.4</ip4>
+        <ip5>10.128.174.5</ip5>
+        <ip6>10.128.174.6</ip6>
+        <ip7>10.128.174.7</ip7>
     </CTRL>
 
     <MN>
@@ -32,8 +36,12 @@
 
     <TEST>
         #Number of times to iterate each case
-        <numIter>3</numIter>
-        <batchIntentSize>50</batchIntentSize>
+        <numIter>10</numIter>
+        <numSwitch>9</numSwitch>
+        <batchThresholdMin>0</batchThresholdMin>
+        <batchThresholdMax>1000</batchThresholdMax>
+        <batchIntentSize>1500</batchIntentSize>
+        <numMult>1</numMult>
     </TEST>
 
     <JSON>
diff --git a/TestON/tests/IntentPerfNext/IntentPerfNext.py b/TestON/tests/IntentPerfNext/IntentPerfNext.py
index dc9fadc..73aa304 100644
--- a/TestON/tests/IntentPerfNext/IntentPerfNext.py
+++ b/TestON/tests/IntentPerfNext/IntentPerfNext.py
@@ -14,6 +14,8 @@
         '''
 
         import time
+        global cluster_count
+        cluster_count = 1 
 
         cell_name = main.params['ENV']['cellName']
 
@@ -23,7 +25,19 @@
         ONOS1_ip = main.params['CTRL']['ip1']
         ONOS2_ip = main.params['CTRL']['ip2']
         ONOS3_ip = main.params['CTRL']['ip3']
-        
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+
+        main.ONOSbench.onos_uninstall(node_ip=ONOS1_ip)
+        main.ONOSbench.onos_uninstall(node_ip=ONOS2_ip)
+        main.ONOSbench.onos_uninstall(node_ip=ONOS3_ip)
+        main.ONOSbench.onos_uninstall(node_ip=ONOS4_ip)
+        main.ONOSbench.onos_uninstall(node_ip=ONOS5_ip)
+        main.ONOSbench.onos_uninstall(node_ip=ONOS6_ip)
+        main.ONOSbench.onos_uninstall(node_ip=ONOS7_ip)
+
         MN1_ip = main.params['MN']['ip1']
         BENCH_ip = main.params['BENCH']['ip']
     
@@ -33,12 +47,16 @@
         cell_file_result = main.ONOSbench.create_cell_file(
                 BENCH_ip, cell_name, MN1_ip,
                 "onos-core,onos-app-metrics,onos-gui",
-                ONOS1_ip, ONOS2_ip, ONOS3_ip)
+                #ONOS1_ip, ONOS2_ip, ONOS3_ip)
+                ONOS1_ip)
 
         main.step("Applying cell file to environment")
         cell_apply_result = main.ONOSbench.set_cell(cell_name)
         verify_cell_result = main.ONOSbench.verify_cell()
 
+        main.step("Removing raft logs")
+        main.ONOSbench.onos_remove_raft_logs()
+
         main.step("Git checkout and pull "+checkout_branch)
         if git_pull == 'on':
             checkout_result = \
@@ -62,27 +80,27 @@
 
         main.step("Installing ONOS package")
         install1_result = main.ONOSbench.onos_install(node=ONOS1_ip)
-        install2_result = main.ONOSbench.onos_install(node=ONOS2_ip)
-        install3_result = main.ONOSbench.onos_install(node=ONOS3_ip)
+        #install2_result = main.ONOSbench.onos_install(node=ONOS2_ip)
+        #install3_result = main.ONOSbench.onos_install(node=ONOS3_ip)
 
         main.step("Set cell for ONOScli env")
         main.ONOS1cli.set_cell(cell_name)
-        main.ONOS2cli.set_cell(cell_name)
-        main.ONOS3cli.set_cell(cell_name)
+        #main.ONOS2cli.set_cell(cell_name)
+        #main.ONOS3cli.set_cell(cell_name)
 
         time.sleep(5)
 
         main.step("Start onos cli")
         cli1 = main.ONOS1cli.start_onos_cli(ONOS1_ip)
-        cli2 = main.ONOS2cli.start_onos_cli(ONOS2_ip)
-        cli3 = main.ONOS3cli.start_onos_cli(ONOS3_ip)
+        #cli2 = main.ONOS2cli.start_onos_cli(ONOS2_ip)
+        #cli3 = main.ONOS3cli.start_onos_cli(ONOS3_ip)
 
         utilities.assert_equals(expect=main.TRUE,
                 actual = cell_file_result and cell_apply_result and\
                          verify_cell_result and checkout_result and\
                          pull_result and build_result and\
-                         install1_result and install2_result and\
-                         install3_result,
+                         install1_result, #and install2_result and\
+                         #install3_result,
                 onpass="ONOS started successfully",
                 onfail="Failed to start ONOS")
 
@@ -355,28 +373,99 @@
         ONOS1_ip = main.params['CTRL']['ip1']
         ONOS2_ip = main.params['CTRL']['ip2']
         ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+       
+        ONOS_ip_list = []
+        for i in range(1, 8):
+            ONOS_ip_list.append(main.params['CTRL']['ip'+str(i)])
+
         ONOS_user = main.params['CTRL']['user']
 
         default_sw_port = main.params['CTRL']['port1']
     
         batch_intent_size = main.params['TEST']['batchIntentSize']
+        batch_thresh_min = int(main.params['TEST']['batchThresholdMin'])
+        batch_thresh_max = int(main.params['TEST']['batchThresholdMax'])
 
         #number of iterations of case
         num_iter = main.params['TEST']['numIter']
+        num_switch = int(main.params['TEST']['numSwitch'])
+        n_thread = main.params['TEST']['numMult']
+
+        #*****
+        global cluster_count
+        #*****
+       
+        #Switch assignment NOTE: hardcoded 
+        if cluster_count == 1:
+            for i in range(1, num_switch+1):
+                main.Mininet1.assign_sw_controller(
+                    sw=str(i), 
+                    ip1=ONOS1_ip,
+                    port1=default_sw_port)
+        if cluster_count == 3:
+            for i in range(1, 3):
+                main.Mininet1.assign_sw_controller(
+                    sw=str(i),
+                    ip1=ONOS1_ip,
+                    port1=default_sw_port)
+            for i in range(3, 6):
+                main.Mininet1.assign_sw_controller(
+                    sw=str(i),
+                    ip1=ONOS2_ip,
+                    port1=default_sw_port)
+            for i in range(6, 9):
+                main.Mininet1.assign_sw_controller(
+                    sw=str(i),
+                    ip1=ONOS3_ip,
+                    port1=default_sw_port)
+        if cluster_count == 5:
+            main.Mininet1.assign_sw_controller(
+                    sw="1",
+                    ip1=ONOS1_ip,
+                    port1=default_sw_port)
+            main.Mininet1.assign_sw_controller(
+                    sw="2",
+                    ip1=ONOS2_ip,
+                    port1=default_sw_port)
+            for i in range(3, 6):
+                main.Mininet1.assign_sw_controller(
+                    sw=str(i),
+                    ip1=ONOS3_ip,
+                    port1=default_sw_port)
+            main.Mininet1.assign_sw_controller(
+                    sw="6",
+                    ip1=ONOS4_ip,
+                    port1=default_sw_port)
+            main.Mininet1.assign_sw_controller(
+                    sw="7",
+                    ip1=ONOS5_ip,
+                    port1=default_sw_port)
+            main.Mininet1.assign_sw_controller(
+                    sw="8",
+                    ip1=ONOS5_ip,
+                    port1=default_sw_port)
         
-        main.Mininet1.assign_sw_controller(
-                sw="1", ip1=ONOS1_ip,port1=default_sw_port)
-        main.Mininet1.assign_sw_controller(
-                sw="2", ip1=ONOS2_ip,port1=default_sw_port)
-        main.Mininet1.assign_sw_controller(
-                sw="3", ip1=ONOS2_ip,port1=default_sw_port)
-        main.Mininet1.assign_sw_controller(
-                sw="4", ip1=ONOS2_ip,port1=default_sw_port)
-        main.Mininet1.assign_sw_controller(
-                sw="5", ip1=ONOS3_ip,port1=default_sw_port)
+        if cluster_count == 7:
+            for i in range(1,9):
+                if i < 7:
+                    main.Mininet1.assign_sw_controller(
+                        sw=str(i),
+                        ip1=ONOS_ip_list[i-1],
+                        port1=default_sw_port)
+                elif i >= 8: 
+                    main.Mininet1.assign_sw_controller(
+                        sw=str(i),
+                        ip1=ONOS_ip_list[6],
+                        port1=default_sw_port)
 
         main.log.report("Batch intent installation test of "+
-                str(batch_intent_size) +" intents")
+               batch_intent_size +" intents")
+
+        batch_result_list = []
 
         main.log.info("Getting list of available devices")
         device_id_list = []
@@ -385,14 +474,121 @@
         for device in json_obj:
             device_id_list.append(device['id'])
 
-        for i in range(0, int(num_iter)):
-            main.log.info("Pushing "+batch_intent_size+" intents")
-            
-            batch_result = main.ONOS1cli.push_test_intents(
-                "of:0000000000000001/1", "of:0000000000000005/2", 
-                batch_intent_size)
+        batch_install_lat = []
+        batch_withdraw_lat = []
+        
+        base_dir = "/tmp/"
 
+        for i in range(0, int(num_iter)):
+            main.log.info("Pushing "+
+                    str(int(batch_intent_size)*int(n_thread))+
+                    " intents. Iteration "+str(i))
+                
+            for node in range(1, cluster_count+1):
+                save_dir = base_dir + "batch_intent_"+str(node)+".txt" 
+                main.ONOSbench.push_test_intents_shell(
+                "of:0000000000000001/"+str(node),
+                "of:0000000000000008/"+str(node),
+                batch_intent_size, save_dir, ONOS_ip_list[node-1],
+                num_mult=n_thread, app_id=node)
+            
+            time.sleep(120 / int(cluster_count))
+            for node in range(1, cluster_count+1):
+                with open(save_dir) as f_onos:
+                    line_count = 0
+                    for line in f_onos:
+                        line = line[1:]
+                        line = line.split(": ")
+                        result = line[1].split(" ")[0]
+                        #TODO: add parameters before appending latency
+                        if line_count == 0:
+                            batch_install_lat.append(result)
+                        elif line_count == 1:
+                            batch_withdraw_lat.append(result)
+                        line_count += 1
+                main.log.info("Batch install latency for ONOS"+
+                    str(node)+" with "+\
+                    str(batch_intent_size) + "intents: "+\
+                    str(batch_install_lat))
             time.sleep(5)
 
+        main.log.report("Max of batch installation latency "+
+            "for ONOS" + str(node)+ ": "+
+            str(max(batch_install_lat)))
+            
+    def CASE5(self,main):
+        '''
+        Increase number of nodes and initiate CLI
+        '''
+        import time
+        import json
+
+        ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS7_ip = main.params['CTRL']['ip7']
+
+        global cluster_count
+        cluster_count += 2
+        main.log.info("Increasing cluster size to "+
+                str(cluster_count))
+
+        install_result = main.FALSE
+
+        if cluster_count == 3:
+            install_result1 = \
+                main.ONOSbench.onos_install(node=ONOS2_ip)
+            install_result2 = \
+                main.ONOSbench.onos_install(node=ONOS3_ip)
+            time.sleep(5)
+
+            main.log.info("Starting ONOS CLI")
+            main.ONOS2cli.start_onos_cli(ONOS2_ip)
+            main.ONOS3cli.start_onos_cli(ONOS3_ip)
+
+            install_result = install_result1 and install_result2
+
+        if cluster_count == 5:
+            main.log.info("Installing ONOS on node 4 and 5")
+            install_result1 = \
+                main.ONOSbench.onos_install(node=ONOS4_ip)
+            install_result2 = \
+                main.ONOSbench.onos_install(node=ONOS5_ip)
+
+            main.log.info("Starting ONOS CLI")
+            main.ONOS4cli.start_onos_cli(ONOS4_ip)
+            main.ONOS5cli.start_onos_cli(ONOS5_ip)
+
+            install_result = install_result1 and install_result2
+
+        if cluster_count == 7:
+            main.log.info("Installing ONOS on node 6 and 7")
+            install_result1 = \
+                main.ONOSbench.onos_install(node=ONOS6_ip)
+            install_result2 = \
+                main.ONOSbench.onos_install(node=ONOS7_ip)
+
+            main.log.info("Starting ONOS CLI")
+            main.ONOS6cli.start_onos_cli(ONOS6_ip)
+            main.ONOS7cli.start_onos_cli(ONOS7_ip)
+
+            install_result = install_result1 and install_result2
+
+        time.sleep(5)
+
+        if install_result == main.TRUE:
+            assertion = main.TRUE
+        else:
+            assertion = main.FALSE
+
+        utilities.assert_equals(expect=main.TRUE, actual=assertion,
+            onpass="Scale out to "+str(cluster_count)+\
+                   " nodes successful",
+            onfail="Scale out to "+str(cluster_count)+\
+                   " nodes failed")
+
 
 
diff --git a/TestON/tests/IntentPerfNext/IntentPerfNext.topo b/TestON/tests/IntentPerfNext/IntentPerfNext.topo
index 75cb259..71df643 100644
--- a/TestON/tests/IntentPerfNext/IntentPerfNext.topo
+++ b/TestON/tests/IntentPerfNext/IntentPerfNext.topo
@@ -36,6 +36,42 @@
             <connect_order>2</connect_order>
             <COMPONENTS> </COMPONENTS>
         </ONOS3cli>
+        
+        <ONOS4cli>
+            <host>10.128.174.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS4cli>
+        
+        <ONOS5cli>
+            <host>10.128.174.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS5cli>
+        
+        <ONOS6cli>
+            <host>10.128.174.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS6cli>
+        
+        <ONOS7cli>
+            <host>10.128.174.10</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS7cli>
 
         <ONOS1>
             <host>10.128.174.1</host>
@@ -53,8 +89,8 @@
             <type>MininetCliDriver</type>
             <connect_order>4</connect_order>
             <COMPONENTS>
-                <arg1> --custom topo-intentFlower.py </arg1>
-                <arg2> --arp --mac --topo mytopo</arg2>
+                <arg1> --custom topo-linear-scaleout.py </arg1>
+                <arg2> --arp --mac --topo mytopo </arg2>
                 <arg3> </arg3>
                 <controller> remote </controller>
             </COMPONENTS>
diff --git a/TestON/tests/TopoConvNext/TopoConvNext.params b/TestON/tests/TopoConvNext/TopoConvNext.params
index 96cb9ad..198befb 100644
--- a/TestON/tests/TopoConvNext/TopoConvNext.params
+++ b/TestON/tests/TopoConvNext/TopoConvNext.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>1,2,3,2,3,2,3,4,2,3,2,3,2,4,2,3,2,3,2</testcases>
+    <testcases>1,2,3,2,3,2,3,2,4,2,3,2,3,2,3,2</testcases>
 
     <ENV>
         <cellName>topo_conv_test</cellName>
@@ -42,10 +42,11 @@
         <onosLogFile>/opt/onos/log/karaf*</onosLogFile>
 
         #Number of times to iterate each case
-        <numIter>5</numIter>
-        <numSwitch1>200</numSwitch1>
+        <numIter>3</numIter>
+        <numSwitch1>500</numSwitch1>
         <numSwitch2>400</numSwitch2>
-        <numSwitch3>500</numSwitch3>
+        <numSwitch3>200</numSwitch3>
+        <numSwitch4>300</numSwitch4>
         #Number of iterations to ignore initially
         <iterIgnore>1</iterIgnore>
 
diff --git a/TestON/tests/TopoConvNext/TopoConvNext.py b/TestON/tests/TopoConvNext/TopoConvNext.py
index 52a951c..68d7b5d 100644
--- a/TestON/tests/TopoConvNext/TopoConvNext.py
+++ b/TestON/tests/TopoConvNext/TopoConvNext.py
@@ -185,6 +185,8 @@
             num_sw = main.params['TEST']['numSwitch2']
         elif topo_iteration == 3:
             num_sw = main.params['TEST']['numSwitch3']
+        elif topo_iteration == 4:
+            num_sw = main.params['TEST']['numSwitch4']
         #***********
 
         #Timestamp 'keys' for json metrics output.
@@ -323,7 +325,7 @@
             #NOTE:
             #       Delay before checking devices to 
             #       help prevent timing out from CLI
-            #       due to multiple command drop
+            #       due to multiple command issuing 
             time.sleep(20)
 
             loop = True
@@ -338,6 +340,9 @@
                 device_json1 = json.loads(device_str1)
                 json_len = len(device_json1) 
                 
+                #NOTE: May want to check the rest of 
+                #      the ONOS instances for device down as well
+
                 for device1 in device_json1: 
                     temp_len = temp_len + 1
                     if device1['available'] == True:
@@ -361,73 +366,10 @@
                     
                         loop = False
                         break
-
-                #if cluster_count == 1:
-                #    device_str1 = main.ONOS1cli.devices(
-                #        node_ip=ONOS_ip_list[1])
-                #    device_json1 = json.loads(device_str1)
-                #    for device1 in device_json1:
-                #        if device1['available'] == False:
-                #            device_count += 1
-                #        else:
-                #            device_count = 0
-                #if cluster_count == 2:
-                #    device_str2 = main.ONOS2cli.devices(
-                #        node_ip=ONOS_ip_list[2])
-                #    device_json2 = json.loads(device_str2)
-                #    for device2 in device_json2:
-                #        if device2['available'] == False:
-                #            device_count += 1
-                #        else:
-                #            device_count = 0
-                #if cluster_count == 3:
-                #    device_str3 = main.ONOS3cli.devices(
-                #        node_ip=ONOS_ip_list[3])
-                #    device_json3 = json.loads(device_str3)
-                #    for device3 in device_json3:
-                #        if device3['available'] == False:
-                #            device_count += 1
-                #        else:
-                #            device_count = 0
-                #if cluster_count == 4:
-                #    device_str4 = main.ONOS4cli.devices(
-                #        node_ip=ONOS_ip_list[4])
-                #    device_json4 = json.loads(device_str4)
-                #    for device4 in device_json4:
-                #        if device4['available'] == False:
-                #            device_count += 1
-                #        else:
-                #            device_count = 0
-                #if cluster_count == 5:
-                #    device_str5 = main.ONOS5cli.devices(
-                #        node_ip=ONOS_ip_list[5])
-                #    device_json5 = json.loads(device_str5)
-                #    for device5 in device_json5:
-                #        if device5['available'] == False:
-                #            device_count += 1
-                #        else:
-                #            device_count = 0
-                #if cluster_count == 7:
-                #    device_str7 = main.ONOS7cli.devices(
-                #        node_ip=ONOS_ip_list[7])
-                #    device_json7 = json.loads(device_str7)
-                #    for device7 in device_json7:
-                #        if device7['available'] == False:
-                #            device_count += 1
-                #        else:
-                #            device_count = 0
-   
-                #If device count is greater than the 
-                #number of switches discovered by all nodes
-                #then remove iptables and measure t0 system time
-                    
                 
                 loop_count += 1
                 time.sleep(1)
 
-            if device_count < int(num_sw):
-                main.log.info("Devices down did not ")
-
             main.log.info("System time t0: "+str(t0_system))
 
             counter_loop = 0
@@ -590,7 +532,7 @@
                         else:
                             main.log.info("Switch discovery latency "+
                                 "exceeded the threshold.")
-                            main.log.info(graph_lat_1)
+                            main.log.info(str(graph_lat_1)+" ms")
                         #Break while loop 
                         break
                 if cluster_count == 2:
@@ -635,7 +577,7 @@
                         else:
                             main.log.info("Switch discovery latency "+
                                 "exceeded the threshold.")
-                            main.log.info(max_graph_lat)
+                            main.log.info(str(max_graph_lat)+" ms")
                         break
                 if cluster_count == 3:
                     if onos1_dev and onos2_dev and onos3_dev:
@@ -693,7 +635,7 @@
                         else:
                             main.log.info("Switch discovery latency "+
                                 "exceeded the threshold.")
-                            main.log.info(max_graph_lat)
+                            main.log.info(str(max_graph_lat)+" ms")
                         
                         break
                 if cluster_count == 4:
@@ -756,7 +698,7 @@
                         else:
                             main.log.info("Switch discovery latency "+
                                 "exceeded the threshold.")
-                            main.log.info(max_graph_lat)
+                            main.log.info(str(max_graph_lat)+" ms")
                 
                         break
                 if cluster_count == 5:
@@ -835,7 +777,7 @@
                         else:
                             main.log.info("Switch discovery latency "+
                                 "exceeded the threshold.")
-                            main.log.info(max_graph_lat)
+                            main.log.info(str(max_graph_lat)+" ms")
                 
                         break
                 if cluster_count == 6:
@@ -918,7 +860,7 @@
                         else:
                             main.log.info("Switch discovery latency "+
                                 "exceeded the threshold.")
-                            main.log.info(max_graph_lat)
+                            main.log.info(str(max_graph_lat)+" ms")
                         
                         break
                 if cluster_count == 7:
@@ -1018,7 +960,7 @@
                         else:
                             main.log.info("Switch discovery latency "+
                                 "exceeded the threshold.")
-                            main.log.info(max_graph_lat)
+                            main.log.info(str(max_graph_lat)+" ms")
                         
                         break
                 
@@ -1153,7 +1095,9 @@
             sw_lat_avg = sum(sw_discovery_lat_list) / \
                      len(sw_discovery_lat_list)
             sw_lat_dev = numpy.std(sw_discovery_lat_list)
-        else: 
+        else:
+            sw_lat_avg = 0
+            sw_lat_dev = 0
             assertion = main.FALSE
         
         main.log.report("Switch connection attempt time avg "+
@@ -1329,3 +1273,6 @@
         utilities.assert_equals(expect=main.TRUE, actual=assertion,
             onpass="Topology size increased successfully",
             onfail="Topology size was not increased")
+
+
+
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.params b/TestON/tests/TopoPerfNext/TopoPerfNext.params
index 42512e8..851522c 100644
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.params
+++ b/TestON/tests/TopoPerfNext/TopoPerfNext.params
@@ -57,7 +57,9 @@
         <linkUpThreshold>0,10000</linkUpThreshold>
         <linkDownThreshold>0,10000</linkDownThreshold>
         <swDisc100Threshold>0,10000</swDisc100Threshold>
-    </TEST>
+    
+        <tabletFile>tablets_3node.json</tabletFile>
+   </TEST>
 
     <JSON>
         <deviceTimestamp>topologyDeviceEventTimestamp</deviceTimestamp>
diff --git a/TestON/tests/TopoPerfNext/TopoPerfNext.py b/TestON/tests/TopoPerfNext/TopoPerfNext.py
index 49089bf..65bc7a9 100644
--- a/TestON/tests/TopoPerfNext/TopoPerfNext.py
+++ b/TestON/tests/TopoPerfNext/TopoPerfNext.py
@@ -47,10 +47,10 @@
         
         main.case("Setting up test environment")
         main.log.info("Copying topology event accumulator config"+\
-            " to ONOS /ppackage/etc")
+            " to ONOS /package/etc")
         main.ONOSbench.handle.sendline("cp ~/"+\
             topo_cfg_file+\
-            "~/ONOS/tools/package/etc/"+\
+            " ~/ONOS/tools/package/etc/"+\
             topo_cfg_name)
         main.ONOSbench.handle.expect("\$")
 
@@ -64,7 +64,7 @@
 
         main.step("Creating cell file")
         cell_file_result = main.ONOSbench.create_cell_file(
-                BENCH_ip, cell_name, MN1_ip, "onos-core",
+                BENCH_ip, cell_name, MN1_ip, "onos-core,onos-app-metrics",
                 ONOS1_ip, ONOS2_ip, ONOS3_ip)
 
         main.step("Applying cell file to environment")
@@ -75,7 +75,8 @@
         #      copy cat log functionality
         main.step("Removing raft/copy-cat logs from ONOS nodes")
         main.ONOSbench.onos_remove_raft_logs()
-        
+        time.sleep(30)
+
         main.step("Git checkout and pull "+checkout_branch)
         if git_pull == 'on':
             checkout_result = \
@@ -114,11 +115,6 @@
         cli2 = main.ONOS2cli.start_onos_cli(ONOS2_ip)
         cli3 = main.ONOS3cli.start_onos_cli(ONOS3_ip)
 
-        main.step("Enable metrics feature")
-        main.ONOS1cli.feature_install("onos-app-metrics")
-        main.ONOS2cli.feature_install("onos-app-metrics")
-        main.ONOS3cli.feature_install("onos-app-metrics")
-
         utilities.assert_equals(expect=main.TRUE,
                 actual= cell_file_result and cell_apply_result and\
                         verify_cell_result and checkout_result and\