WIP: Dynamic switch assignment for scale-out scenarios and scale-out test
diff --git a/TestON/tests/TopoConvNext/TopoConvNext.py b/TestON/tests/TopoConvNext/TopoConvNext.py
index 481a726..874eb41 100644
--- a/TestON/tests/TopoConvNext/TopoConvNext.py
+++ b/TestON/tests/TopoConvNext/TopoConvNext.py
@@ -13,16 +13,17 @@
     def __init__(self):
         self.default = ''
 
-    #******
-    #Global cluster count for scale-out purposes
-    cluster_count = 1
-    #******
-
     def CASE1(self, main):
         '''
         ONOS startup sequence
         '''
         import time
+
+        #******
+        #Global cluster count for scale-out purposes
+        global cluster_count 
+        cluster_count = 1
+        #******
     
         cell_name = main.params['ENV']['cellName']
 
@@ -30,6 +31,12 @@
         checkout_branch = main.params['GIT']['checkout']
 
         ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS7_ip = main.params['CTRL']['ip7']
         MN1_ip = main.params['MN']['ip1']
         BENCH_ip = main.params['BENCH']['ip']
 
@@ -39,7 +46,8 @@
         main.step("Creating cell file")
         cell_file_result = main.ONOSbench.create_cell_file(
                 BENCH_ip, cell_name, MN1_ip, "onos-core",
-                ONOS1_ip)
+                ONOS1_ip, ONOS2_ip, ONOS3_ip, ONOS4_ip,
+                ONOS5_ip, ONOS6_ip, ONOS7_ip)
 
         main.step("Applying cell file to environment")
         cell_apply_result = main.ONOSbench.set_cell(cell_name)
@@ -61,10 +69,17 @@
 
         main.step("Set cell for ONOS cli env")
         main.ONOS1cli.set_cell(cell_name)
-
+        main.ONOS2cli.set_cell(cell_name)
+        main.ONOS3cli.set_cell(cell_name)
+        main.ONOS4cli.set_cell(cell_name)
+        main.ONOS5cli.set_cell(cell_name)
+        main.ONOS6cli.set_cell(cell_name)
+        main.ONOS7cli.set_cell(cell_name)
+    
         main.step("Creating ONOS package")
         package_result = main.ONOSbench.onos_package()
 
+        #Start test with single node only
         main.step("Installing ONOS package")
         install1_result = main.ONOSbench.onos_install(node=ONOS1_ip)
 
@@ -80,8 +95,7 @@
                 actual= cell_file_result and cell_apply_result and\
                         verify_cell_result and checkout_result and\
                         pull_result and mvn_result and\
-                        install1_result and install2_result and\
-                        install3_result,
+                        install1_result,
                 onpass="Test Environment setup successful",
                 onfail="Failed to setup test environment")
     
@@ -107,13 +121,13 @@
         import json
 
         ONOS_ip_list = []
-        ONOS_ip_list.append(main.params['ONOS']['ip1'])
-        ONOS_ip_list.append(main.params['ONOS']['ip2'])
-        ONOS_ip_list.append(main.params['ONOS']['ip3'])
-        ONOS_ip_list.append(main.params['ONOS']['ip4'])
-        ONOS_ip_list.append(main.params['ONOS']['ip5'])
-        ONOS_ip_list.append(main.params['ONOS']['ip6'])
-        ONOS_ip_list.append(main.params['ONOS']['ip7'])
+        ONOS_ip_list.append(main.params['CTRL']['ip1'])
+        ONOS_ip_list.append(main.params['CTRL']['ip2'])
+        ONOS_ip_list.append(main.params['CTRL']['ip3'])
+        ONOS_ip_list.append(main.params['CTRL']['ip4'])
+        ONOS_ip_list.append(main.params['CTRL']['ip5'])
+        ONOS_ip_list.append(main.params['CTRL']['ip6'])
+        ONOS_ip_list.append(main.params['CTRL']['ip7'])
         MN1_ip = main.params['MN']['ip1']
         ONOS_user = main.params['CTRL']['user']
 
@@ -134,8 +148,10 @@
         sw_disc_threshold_min = int(sw_disc_threshold_obj[0])
         sw_disc_threshold_max = int(sw_disc_threshold_obj[1])
 
-        tshark_ofp_output = "/tmp/tshark_ofp_"+num_sw+"sw.txt"
-        tshark_tcp_output = "/tmp/tshark_tcp_"+num_sw+"sw.txt"
+        tshark_ofp_output = \
+                "/tmp/tshark_ofp_"+num_sw+"sw_"+cluster_count+".txt"
+        tshark_tcp_output = \
+                "/tmp/tshark_tcp_"+num_sw+"sw_"+cluster_count+".txt"
 
         tshark_ofp_result_list = []
         tshark_tcp_result_list = []
@@ -146,19 +162,49 @@
        
         main.log.report("Currently active ONOS node(s): ")
         report_str = "Node "
-        for node in cluster_count:
+        for node in range(0, cluster_count):
             report_str += (str(node+1) + " ") 
         main.log.report(report_str)
         
         main.step("Assigning "+num_sw+" switches to each ONOS")
         index = 1 
-        for node in cluster_count:
+        for node in range(0, cluster_count):
             for i in range(index, int(num_sw)+index):
                 main.Mininet1.assign_sw_controller(
                         sw=str(i),
                         ip1=ONOS_ip_list[node],
                         port1=default_sw_port)
-            index = i 
+            index = i+1 
+
+        main.log.info("Please check ptpd configuration to ensure "+\
+                "all nodes' system times are in sync")
+        main.log.info("Checking system time across nodes")
+        os.system("echo TestON system time:  $(($(date +%s%N)/1000000))")
+        main.ONOS1.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS2.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS3.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS4.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS5.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS6.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS7.handle.sendline("echo $(($(date +%s%N)/1000000))")
+
+        main.ONOS1.handle.expect("0))")
+        main.log.info("ONOS1 time: "+main.ONOS1.handle.after)
+        main.ONOS2.handle.expect("0))")
+        main.log.info("ONOS2 time: "+main.ONOS2.handle.after)
+        main.ONOS3.handle.expect("0))")
+        main.log.info("ONOS3 time: "+main.ONOS3.handle.after)
+        main.ONOS4.handle.expect("0))")
+        main.log.info("ONOS4 time: "+main.ONOS4.handle.after)
+        main.ONOS5.handle.expect("0))")
+        main.log.info("ONOS5 time: "+main.ONOS5.handle.after)
+        main.ONOS6.handle.expect("0))")
+        main.log.info("ONOS6 time: "+main.ONOS6.handle.after)
+        main.ONOS7.handle.expect("0))")
+        main.log.info("ONOS7 time: "+main.ONOS7.handle.after)
+        
+        time.sleep(10)
+
 
         #TODO: Implement modular switch discovery measurements
         #for scale-out scenario
@@ -166,7 +212,7 @@
 
     def CASE3(self, main):
         '''
-        Increase number of nodes and start those nodes
+        Increase number of nodes and initiate CLI
         '''
         import time
         import subprocess
@@ -175,13 +221,13 @@
         import json
        
         ONOS_ip_list = []
-        ONOS_ip_list.append(main.params['ONOS']['ip1'])
-        ONOS_ip_list.append(main.params['ONOS']['ip2'])
-        ONOS_ip_list.append(main.params['ONOS']['ip3'])
-        ONOS_ip_list.append(main.params['ONOS']['ip4'])
-        ONOS_ip_list.append(main.params['ONOS']['ip5'])
-        ONOS_ip_list.append(main.params['ONOS']['ip6'])
-        ONOS_ip_list.append(main.params['ONOS']['ip7'])
+        ONOS_ip_list.append(main.params['CTRL']['ip1'])
+        ONOS_ip_list.append(main.params['CTRL']['ip2'])
+        ONOS_ip_list.append(main.params['CTRL']['ip3'])
+        ONOS_ip_list.append(main.params['CTRL']['ip4'])
+        ONOS_ip_list.append(main.params['CTRL']['ip5'])
+        ONOS_ip_list.append(main.params['CTRL']['ip6'])
+        ONOS_ip_list.append(main.params['CTRL']['ip7'])
         
         MN1_ip = main.params['MN']['ip1']
         BENCH_ip = main.params['BENCH']['ip']
@@ -193,10 +239,10 @@
         #test steps of node sizes, such as 1,3,5,7
         
         global cluster_count
-        cluster_count += 1
+        cluster_count += 2 
        
         #Supports up to 7 node configuration
-        for node in cluster_count:
+        for node in range(0, cluster_count):
             main.log.info("Installing ONOS instance: "+
                     ONOS_ip_list[node])
             main.ONOSbench.onos_install(ONOS_ip_list[node])