WIP: Dynamic switch assignment for scale-out scenarios and scale-out test
diff --git a/TestON/tests/TopoConvNext/TopoConvNext.params b/TestON/tests/TopoConvNext/TopoConvNext.params
index 67af5ec..67ad10f 100644
--- a/TestON/tests/TopoConvNext/TopoConvNext.params
+++ b/TestON/tests/TopoConvNext/TopoConvNext.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>1,2,3</testcases>
+    <testcases>1,2</testcases>
 
     <ENV>
         <cellName>topo_conv_test</cellName>
diff --git a/TestON/tests/TopoConvNext/TopoConvNext.py b/TestON/tests/TopoConvNext/TopoConvNext.py
index 481a726..874eb41 100644
--- a/TestON/tests/TopoConvNext/TopoConvNext.py
+++ b/TestON/tests/TopoConvNext/TopoConvNext.py
@@ -13,16 +13,17 @@
     def __init__(self):
         self.default = ''
 
-    #******
-    #Global cluster count for scale-out purposes
-    cluster_count = 1
-    #******
-
     def CASE1(self, main):
         '''
         ONOS startup sequence
         '''
         import time
+
+        #******
+        #Global cluster count for scale-out purposes
+        global cluster_count 
+        cluster_count = 1
+        #******
     
         cell_name = main.params['ENV']['cellName']
 
@@ -30,6 +31,12 @@
         checkout_branch = main.params['GIT']['checkout']
 
         ONOS1_ip = main.params['CTRL']['ip1']
+        ONOS2_ip = main.params['CTRL']['ip2']
+        ONOS3_ip = main.params['CTRL']['ip3']
+        ONOS4_ip = main.params['CTRL']['ip4']
+        ONOS5_ip = main.params['CTRL']['ip5']
+        ONOS6_ip = main.params['CTRL']['ip6']
+        ONOS7_ip = main.params['CTRL']['ip7']
         MN1_ip = main.params['MN']['ip1']
         BENCH_ip = main.params['BENCH']['ip']
 
@@ -39,7 +46,8 @@
         main.step("Creating cell file")
         cell_file_result = main.ONOSbench.create_cell_file(
                 BENCH_ip, cell_name, MN1_ip, "onos-core",
-                ONOS1_ip)
+                ONOS1_ip, ONOS2_ip, ONOS3_ip, ONOS4_ip,
+                ONOS5_ip, ONOS6_ip, ONOS7_ip)
 
         main.step("Applying cell file to environment")
         cell_apply_result = main.ONOSbench.set_cell(cell_name)
@@ -61,10 +69,17 @@
 
         main.step("Set cell for ONOS cli env")
         main.ONOS1cli.set_cell(cell_name)
-
+        main.ONOS2cli.set_cell(cell_name)
+        main.ONOS3cli.set_cell(cell_name)
+        main.ONOS4cli.set_cell(cell_name)
+        main.ONOS5cli.set_cell(cell_name)
+        main.ONOS6cli.set_cell(cell_name)
+        main.ONOS7cli.set_cell(cell_name)
+    
         main.step("Creating ONOS package")
         package_result = main.ONOSbench.onos_package()
 
+        #Start test with single node only
         main.step("Installing ONOS package")
         install1_result = main.ONOSbench.onos_install(node=ONOS1_ip)
 
@@ -80,8 +95,7 @@
                 actual= cell_file_result and cell_apply_result and\
                         verify_cell_result and checkout_result and\
                         pull_result and mvn_result and\
-                        install1_result and install2_result and\
-                        install3_result,
+                        install1_result,
                 onpass="Test Environment setup successful",
                 onfail="Failed to setup test environment")
     
@@ -107,13 +121,13 @@
         import json
 
         ONOS_ip_list = []
-        ONOS_ip_list.append(main.params['ONOS']['ip1'])
-        ONOS_ip_list.append(main.params['ONOS']['ip2'])
-        ONOS_ip_list.append(main.params['ONOS']['ip3'])
-        ONOS_ip_list.append(main.params['ONOS']['ip4'])
-        ONOS_ip_list.append(main.params['ONOS']['ip5'])
-        ONOS_ip_list.append(main.params['ONOS']['ip6'])
-        ONOS_ip_list.append(main.params['ONOS']['ip7'])
+        ONOS_ip_list.append(main.params['CTRL']['ip1'])
+        ONOS_ip_list.append(main.params['CTRL']['ip2'])
+        ONOS_ip_list.append(main.params['CTRL']['ip3'])
+        ONOS_ip_list.append(main.params['CTRL']['ip4'])
+        ONOS_ip_list.append(main.params['CTRL']['ip5'])
+        ONOS_ip_list.append(main.params['CTRL']['ip6'])
+        ONOS_ip_list.append(main.params['CTRL']['ip7'])
         MN1_ip = main.params['MN']['ip1']
         ONOS_user = main.params['CTRL']['user']
 
@@ -134,8 +148,10 @@
         sw_disc_threshold_min = int(sw_disc_threshold_obj[0])
         sw_disc_threshold_max = int(sw_disc_threshold_obj[1])
 
-        tshark_ofp_output = "/tmp/tshark_ofp_"+num_sw+"sw.txt"
-        tshark_tcp_output = "/tmp/tshark_tcp_"+num_sw+"sw.txt"
+        tshark_ofp_output = \
+                "/tmp/tshark_ofp_"+num_sw+"sw_"+cluster_count+".txt"
+        tshark_tcp_output = \
+                "/tmp/tshark_tcp_"+num_sw+"sw_"+cluster_count+".txt"
 
         tshark_ofp_result_list = []
         tshark_tcp_result_list = []
@@ -146,19 +162,49 @@
        
         main.log.report("Currently active ONOS node(s): ")
         report_str = "Node "
-        for node in cluster_count:
+        for node in range(0, cluster_count):
             report_str += (str(node+1) + " ") 
         main.log.report(report_str)
         
         main.step("Assigning "+num_sw+" switches to each ONOS")
         index = 1 
-        for node in cluster_count:
+        for node in range(0, cluster_count):
             for i in range(index, int(num_sw)+index):
                 main.Mininet1.assign_sw_controller(
                         sw=str(i),
                         ip1=ONOS_ip_list[node],
                         port1=default_sw_port)
-            index = i 
+            index = i+1 
+
+        main.log.info("Please check ptpd configuration to ensure "+\
+                "all nodes' system times are in sync")
+        main.log.info("Checking system time across nodes")
+        os.system("echo TestON system time:  $(($(date +%s%N)/1000000))")
+        main.ONOS1.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS2.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS3.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS4.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS5.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS6.handle.sendline("echo $(($(date +%s%N)/1000000))")
+        main.ONOS7.handle.sendline("echo $(($(date +%s%N)/1000000))")
+
+        main.ONOS1.handle.expect("0))")
+        main.log.info("ONOS1 time: "+main.ONOS1.handle.after)
+        main.ONOS2.handle.expect("0))")
+        main.log.info("ONOS2 time: "+main.ONOS2.handle.after)
+        main.ONOS3.handle.expect("0))")
+        main.log.info("ONOS3 time: "+main.ONOS3.handle.after)
+        main.ONOS4.handle.expect("0))")
+        main.log.info("ONOS4 time: "+main.ONOS4.handle.after)
+        main.ONOS5.handle.expect("0))")
+        main.log.info("ONOS5 time: "+main.ONOS5.handle.after)
+        main.ONOS6.handle.expect("0))")
+        main.log.info("ONOS6 time: "+main.ONOS6.handle.after)
+        main.ONOS7.handle.expect("0))")
+        main.log.info("ONOS7 time: "+main.ONOS7.handle.after)
+        
+        time.sleep(10)
+
 
         #TODO: Implement modular switch discovery measurements
         #for scale-out scenario
@@ -166,7 +212,7 @@
 
     def CASE3(self, main):
         '''
-        Increase number of nodes and start those nodes
+        Increase number of nodes and initiate CLI
         '''
         import time
         import subprocess
@@ -175,13 +221,13 @@
         import json
        
         ONOS_ip_list = []
-        ONOS_ip_list.append(main.params['ONOS']['ip1'])
-        ONOS_ip_list.append(main.params['ONOS']['ip2'])
-        ONOS_ip_list.append(main.params['ONOS']['ip3'])
-        ONOS_ip_list.append(main.params['ONOS']['ip4'])
-        ONOS_ip_list.append(main.params['ONOS']['ip5'])
-        ONOS_ip_list.append(main.params['ONOS']['ip6'])
-        ONOS_ip_list.append(main.params['ONOS']['ip7'])
+        ONOS_ip_list.append(main.params['CTRL']['ip1'])
+        ONOS_ip_list.append(main.params['CTRL']['ip2'])
+        ONOS_ip_list.append(main.params['CTRL']['ip3'])
+        ONOS_ip_list.append(main.params['CTRL']['ip4'])
+        ONOS_ip_list.append(main.params['CTRL']['ip5'])
+        ONOS_ip_list.append(main.params['CTRL']['ip6'])
+        ONOS_ip_list.append(main.params['CTRL']['ip7'])
         
         MN1_ip = main.params['MN']['ip1']
         BENCH_ip = main.params['BENCH']['ip']
@@ -193,10 +239,10 @@
         #test steps of node sizes, such as 1,3,5,7
         
         global cluster_count
-        cluster_count += 1
+        cluster_count += 2 
        
         #Supports up to 7 node configuration
-        for node in cluster_count:
+        for node in range(0, cluster_count):
             main.log.info("Installing ONOS instance: "+
                     ONOS_ip_list[node])
             main.ONOSbench.onos_install(ONOS_ip_list[node]) 
diff --git a/TestON/tests/TopoConvNext/TopoConvNext.topo b/TestON/tests/TopoConvNext/TopoConvNext.topo
index b4b20f0..855e688 100644
--- a/TestON/tests/TopoConvNext/TopoConvNext.topo
+++ b/TestON/tests/TopoConvNext/TopoConvNext.topo
@@ -99,13 +99,49 @@
             <connect_order>11</connect_order>
             <COMPONENTS> </COMPONENTS>
         </ONOS3>
+        
+        <ONOS4>
+            <host>10.128.174.4</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>12</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS4>
+
+        <ONOS5>
+            <host>10.128.174.5</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>13</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS5>
+    
+        <ONOS6>
+            <host>10.128.174.6</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>14</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS6>
+        
+        <ONOS7>
+            <host>10.128.174.7</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosDriver</type>
+            <connect_order>15</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS7>
 
         <Mininet1>
             <host>10.128.10.90</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>MininetCliDriver</type>
-            <connect_order>12</connect_order>
+            <connect_order>16</connect_order>
             <COMPONENTS>
                 <arg1> --custom topo-700sw.py </arg1>
                 <arg2> --arp --mac --topo mytopo</arg2>
@@ -119,7 +155,7 @@
             <user>admin</user>
             <password>onos_test</password>
             <type>RemoteMininetDriver</type>
-            <connect_order>13</connect_order>
+            <connect_order>17</connect_order>
             <COMPONENTS> </COMPONENTS>
         </Mininet2>