Driver fixes, Device device discovery in Sanity, two case additions to Perf
diff --git a/TestON/tests/OnosPerf4nodes/OnosPerf4nodes.py b/TestON/tests/OnosPerf4nodes/OnosPerf4nodes.py
index 8d65f35..f84b232 100644
--- a/TestON/tests/OnosPerf4nodes/OnosPerf4nodes.py
+++ b/TestON/tests/OnosPerf4nodes/OnosPerf4nodes.py
@@ -14,12 +14,28 @@
         import time
         main.log.report("Checking if the startup was clean") 
         main.case("Checking if the startup was clean...")
+        #main.ONOS1.stop()
+        #main.ONOS2.stop()
+        #main.ONOS3.stop()
+        #main.ONOS4.stop()
         main.step("Testing startup Zookeeper")
         main.ONOS1.get_version()
         data =  main.Zookeeper1.isup()
         utilities.assert_equals(expect=main.TRUE,actual=data,onpass="Zookeeper is up!",onfail="Zookeeper is down...")
         main.step("Testing startup Cassandra")
         data =  main.Cassandra1.isup()
+        if data == main.FALSE:
+            main.Cassandra1.stop()
+            main.Cassandra2.stop()
+            main.Cassandra3.stop()
+            main.Cassandra4.stop()
+
+            time.sleep(5)
+
+            main.Cassandra1.start()
+            main.Cassandra2.start()
+            main.Cassandra3.start()
+            main.Cassandra4.start()
         utilities.assert_equals(expect=main.TRUE,actual=data,onpass="Cassandra is up!",onfail="Cassandra is down...")
         main.ONOS1.drop_keyspace()
         time.sleep(5) 
@@ -41,6 +57,10 @@
         data = data & main.ONOS3.isup()
         data = data & main.ONOS4.isup()
         main.ONOS1.start_rest()
+        time.sleep(5)
+        test= main.ONOS1.rest_status()
+        if test == main.FALSE:
+            main.ONOS1.start_rest()
         #topoview = main.ONOS1.check_status(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
 
         utilities.assert_equals(expect=main.TRUE,actual=data,onpass="ONOS is up and running ",onfail="ONOS didn't start ...")
@@ -54,19 +74,67 @@
         import time
         main.step("Checking if MN switches exist")
         main.log.report("Check if MN switches exist")
-        result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
-        for i in range(2):
-            if result == main.FALSE: 
-                time.sleep(5)
-                result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
-            else: 
-                break
-        main.step("Verifying the result")
-        utilities.assert_equals(expect=main.TRUE,actual=result,onpass="MN switches exist",onfail="MN is missing switches and or links...")
+        #result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
+        #for i in range(2):
+        #    if result == main.FALSE: 
+        #        time.sleep(5)
+        #        result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
+        #    else: 
+        #        break
+        #main.step("Verifying the result")
+        #utilities.assert_equals(expect=main.TRUE,actual=result,onpass="MN switches exist",onfail="MN is missing switches and or links...")
 
+        main.step("assigning ONOS controllers to switches")
+        for i in range(25):
+            if i < 3:
+                j=i+1
+                main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'])
+                time.sleep(1)
+                main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'])
+            elif i >= 3 and i < 5:
+                j=i+1
+                main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip2'],port1=main.params['CTRL']['port2'])
+                time.sleep(1)
+                main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'])
+            elif i >= 5 and i < 15:
+                j=i+1
+                main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip3'],port1=main.params['CTRL']['port3'])
+                time.sleep(1)
+                main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'])
+            else:
+                j=i+16
+                main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip4'],port1=main.params['CTRL']['port4'])
+                time.sleep(1)
+                main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'])
+        main.Mininet1.get_sw_controller("s1")
 
     def CASE3(self,main) :
         '''
+        Verifies that ONOS sees the right topology... 
+        '''
+        import time
+        main.log.report("checking if ONOS sees the right topo...") 
+        main.case("TOPO check")
+        main.step("calling rest calls") 
+        for i in range(9):
+            result = main.ONOS1.check_status(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
+            time.sleep(5)
+            if result == 1:
+                break
+        if result == 0:
+            main.ONOS1.start()
+            main.ONOS2.start()
+            main.ONOS3.start()
+            main.ONOS4.start()
+            for i in range(9):
+                result = main.ONOS1.check_status(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
+                time.sleep(5)
+                if result == 1:
+                    break
+        utilities.assert_equals(expect=1,actual=result)
+       
+    def CASE4(self,main) :
+        '''
         This Test case: 
             - Clears out any leftover flows
             - Adds new flows into ONOS
@@ -86,14 +154,39 @@
             result = main.ONOS1.check_flow()
             if result == main.TRUE: 
                 t2 = time.time()
-                main.log.info( 'Adding flows took %0.3f ms' % ((t2-t1)*1000.0))
+                main.log.info( 'Adding flows took %0.3f s' % (t2-t1))
                 break
             time.sleep(5)
             main.log.info("Checking Flows again...")
+
+        strtTime = time.time()
+        count = 1
+        i = 6
+        while i < 16 :
+            main.log.info("\n\t\t\t\th"+str(i)+" IS PINGING h"+str(i+25) )
+            ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25))
+            if ping == main.FALSE and count < 3:
+                count = count + 1
+                i = 6
+                main.log.info("Ping failed, making attempt number "+str(count)+" in 10 seconds")
+                time.sleep(10)
+            elif ping == main.FALSE and count ==3:
+                main.log.error("Ping test failed")
+                i = 17
+                result = main.FALSE
+            elif ping == main.TRUE:
+                i = i + 1
+                result = main.TRUE
+        endTime = time.time()
+        if result == main.TRUE:
+            main.log.info("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds")
+        else:
+            main.log.info("\tPING TEST FAIL")
+
         utilities.assert_equals(expect=main.TRUE,actual=result,onpass="flows are good",onfail="FLOWS not correct") 
 
 
-    def CASE4(self,main) :
+    def CASE5(self,main) :
         '''
         Tests a single ping 
         '''
@@ -103,12 +196,13 @@
         utilities.assert_equals(expect=main.TRUE,actual=ping_result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE") 
 
 
-    def CASE5(self,main) :
+    def CASE6(self,main) :
         '''
         Starts continuous pings on the Mininet nodes
         '''
-        main.log.report("Starting continuous ping")
+        main.log.report("Starting continuous ping, then toggle a single link in the center triangle")
         import time
+        import os
         main.case("Starting long ping... ") 
         main.Mininet4.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'])
         main.Mininet4.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'])
@@ -120,7 +214,7 @@
         main.Mininet4.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'])
         main.Mininet4.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'])
         main.Mininet4.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'])
-        time.sleep(5) 
+        main.step("Check that the pings are going") 
         result = main.Mininet4.pingstatus(src=main.params['PING']['source1'])
         result = result & main.Mininet4.pingstatus(src=main.params['PING']['source2'])
         result = result & main.Mininet4.pingstatus(src=main.params['PING']['source3'])
@@ -131,33 +225,18 @@
         result = result & main.Mininet4.pingstatus(src=main.params['PING']['source8'])
         result = result & main.Mininet4.pingstatus(src=main.params['PING']['source9'])
         result = result & main.Mininet4.pingstatus(src=main.params['PING']['source10'])
-        utilities.assert_equals(expect=main.TRUE,actual=result) 
-
-
-    def CASE6(self,main) : 
-        '''
-        Brings the defined link down and up a certain number of iterations
-        Stops the continuous pings that are running on the mininet machines from CASE 5
-        '''
-        import time
-        import os
-        main.log.report("Links UP/DOWN and stop ping")
-        main.log.case( "Link down number of iterations: " +  main.params['Iterations'] )
-        os.popen("echo "" > ~/tools/downtimes.log")
+        main.step( "Link down number of iterations: " +  main.params['Iterations'] )
         for i in range(int(main.params['Iterations'])):
-            main.log.info("Bringing Link down... ")
-            #main.ONOS1.link_down(SDPID=main.params['LINK1']['dpid1'],SPORT=main.params['LINK1']['port1'],DDPID=main.params['LINK1']['dpid2'],DPORT=main.params['LINK1']['port2'])
-            main.Mininet1.link(END1=main.params['LINK']['begin'],END2=main.params['LINK']['end'],OPTION="down")
-            os.popen("echo `date +%s` >> ~/tools/downtimes.log")
+            main.log.info("Bringing Link down... ") 
+            main.Mininet1.link(END1="s1",END2="s2",OPTION="down")
             main.log.info( "Waiting " + main.params['WaitTime'] + " seconds.... " )
             time.sleep( int(main.params['WaitTime']) )
             main.log.info("Bringing Link up... ")
-            #main.ONOS1.link_up(SDPID=main.params['LINK1']['dpid1'],SPORT=main.params['LINK1']['port1'],DDPID=main.params['LINK1']['dpid2'],DPORT=main.params['LINK1']['port2'])
-            main.Mininet1.link(END1=main.params['LINK']['begin'],END2=main.params['LINK']['end'],OPTION="up")
+            main.Mininet1.link(END1="s1",END2="s2",OPTION="up")
             main.log.info( "Waiting " + main.params['WaitTime'] + " seconds.... " )
             time.sleep( int(main.params['WaitTime']) )
         main.case("Killing remote ping processes ") 
-        result = main.Mininet4.pingKill() 
+        result =  result & main.Mininet4.pingKill() 
         utilities.assert_equals(expect=main.TRUE,actual=result) 
        
 
@@ -170,16 +249,193 @@
         main.log.report("Process ping data")
         main.case("Processing Ping data") 
         time.sleep(3) 
-        result=os.popen("/home/admin/tools/shell.sh " + main.params['Iterations']).read()
-        average=result.split(":")[1] 
-        main.log.info( "Reroute times are... " ) 
-        main.log.report( result ) 
+        #result=os.popen("/home/admin/tools/shell.sh " + main.params['Iterations']).read()
+        try:
+            result=os.popen("/home/admin/get_reroute_times.py").read() 
+            average=result.split(":")[1] 
+            main.log.info( "Reroute times are... " ) 
+            main.log.report( result + " seconds" ) 
+            try:
+                if float(average) < float(main.params['TargetTime']) :
+                    test=main.TRUE
+                else:
+                    test=main.FALSE
+            except ValueError: 
+                main.log.error("Data is corrupted")
+                test=main.FALSE
+        except:
+            main.log.report("No data")
+            test=main.FALSE
+        utilities.assert_equals(expect=main.TRUE,actual=test,onpass="Average is less then the target time!",onfail="Average is worse then target time... ")
+
+    def CASE8(self,main) :
+        '''
+        Starts continuous pings on the Mininet nodes
+        '''
+        main.log.report("Start continuous pings, then toggle multiple links in center triangle")
+        import time
+        import os
+
+        main.case("Starting long ping... ")
+        main.Mininet4.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'])
+        main.Mininet4.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'])
+        main.Mininet4.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'])
+        main.Mininet4.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'])
+        main.Mininet4.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'])
+        main.Mininet4.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'])
+        main.Mininet4.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'])
+        main.Mininet4.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'])
+        main.Mininet4.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'])
+        main.Mininet4.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'])
+        main.step("Check that the pings are going")
+        result = main.Mininet4.pingstatus(src=main.params['PING']['source1'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source2'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source3'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source4'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source5'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source6'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source7'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source8'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source9'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source10'])
+        main.step( "Making topo change while flows are rerouting")
+        main.step( "Link down number of iterations: " +  main.params['Iterations'] )
+        for i in range(int(main.params['Iterations'])):
+            main.log.info("s1-s2 link down")
+            main.Mininet1.link(END1="s1",END2="s2",OPTION="down")
+            main.Mininet1.link(END1="s1",END2="s3",OPTION="up")
+            main.Mininet1.link(END1="s2",END2="s3",OPTION="up")
+
+            time.sleep(5)
+
+            main.log.info("s1-s2 link up | s1-s3 link down | s2-s3 link down")
+            main.Mininet1.link(END1="s1",END2="s2",OPTION="up")
+            main.Mininet1.link(END1="s1",END2="s3",OPTION="down")
+            main.Mininet1.link(END1="s2",END2="s3",OPTION="down")
+
+            main.log.info( "Waiting " + main.params['WaitTime'] + " seconds.... " )
+            time.sleep( int(main.params['WaitTime']) )
+
+        main.case("Killing remote ping processes ")
+        result =  result & main.Mininet4.pingKill()
+        utilities.assert_equals(expect=main.TRUE,actual=result)
+
+    def CASE9(self,main) :
+        '''
+        Processes all of the ping data and outputs raw data and an overall average
+        '''
+        import os
+        import time
+        main.log.report("Process ping data")
+        main.case("Processing Ping data")
+        time.sleep(3)
+        #result=os.popen("/home/admin/tools/shell.sh " + main.params['Iterations']).read()
+        result=os.popen("/home/admin/get_reroute_times.py").read()
+        average=result.split(":")[1]
+        main.log.info( "Reroute times are... " )
+        main.log.report( result )
         try:
             if float(average) < float(main.params['TargetTime']) :
                 test=main.TRUE
             else:
                 test=main.FALSE
-        except ValueError: 
+        except ValueError:
+            main.log.error("Data is corrupted")
+            test=main.FALSE
+        utilities.assert_equals(expect=main.TRUE,actual=test,onpass="Average is less then the target time!",onfail="Average is worse then target time... ")
+
+    def CASE10(self,main) :
+        '''
+        Starts continuous pings on the Mininet nodes
+        '''
+        main.log.info("Make sure all links in triangle are up")
+        main.Mininet1.link(END1="s1",END2="s2",OPTION="up")
+        main.Mininet1.link(END1="s1",END2="s3",OPTION="up")
+        main.Mininet1.link(END1="s2",END2="s3",OPTION="up")
+        main.log.report("Start continuous pings, then toggle one link in center triangle and start/stop 1 ONOS node")
+        import time
+        import os
+
+        main.case("Starting long ping... ")
+        main.Mininet4.pingLong(src=main.params['PING']['source1'],target=main.params['PING']['target1'])
+        main.Mininet4.pingLong(src=main.params['PING']['source2'],target=main.params['PING']['target2'])
+        main.Mininet4.pingLong(src=main.params['PING']['source3'],target=main.params['PING']['target3'])
+        main.Mininet4.pingLong(src=main.params['PING']['source4'],target=main.params['PING']['target4'])
+        main.Mininet4.pingLong(src=main.params['PING']['source5'],target=main.params['PING']['target5'])
+        main.Mininet4.pingLong(src=main.params['PING']['source6'],target=main.params['PING']['target6'])
+        main.Mininet4.pingLong(src=main.params['PING']['source7'],target=main.params['PING']['target7'])
+        main.Mininet4.pingLong(src=main.params['PING']['source8'],target=main.params['PING']['target8'])
+        main.Mininet4.pingLong(src=main.params['PING']['source9'],target=main.params['PING']['target9'])
+        main.Mininet4.pingLong(src=main.params['PING']['source10'],target=main.params['PING']['target10'])
+        main.step("Check that the pings are going")
+        result = main.Mininet4.pingstatus(src=main.params['PING']['source1'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source2'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source3'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source4'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source5'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source6'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source7'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source8'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source9'])
+        result = result & main.Mininet4.pingstatus(src=main.params['PING']['source10'])
+        main.step( "Making topo change while flows are rerouting")
+        main.step( "Link down number of iterations: " +  main.params['Iterations'] )
+        for i in range(int(main.params['Iterations'])):
+            main.log.info("s1-s2 link down | Onos 1 down")
+            main.Mininet1.link(END1="s1",END2="s2",OPTION="down")
+            if i % 4 == 0:
+                main.ONOS1.stop()
+            elif i % 4 == 1:
+                main.ONOS2.stop()
+            elif i % 4 == 2:
+                main.ONOS3.stop()
+            else:
+                main.ONOS4.stop()
+
+            time.sleep(5)
+
+            main.log.info("s1-s2 link up | Onos 1 back up")
+            main.Mininet1.link(END1="s1",END2="s2",OPTION="up")
+            if i % 4 == 0:
+                main.ONOS1.start()
+                main.ONOS1.isup()
+            elif i % 4 == 1:
+                main.ONOS2.start()
+                main.ONOS2.isup()
+            elif i % 4 == 2:
+                main.ONOS3.start()
+                main.ONOS3.isup()
+            else:
+                main.ONOS4.start()
+                main.ONOS4.isup()
+
+            main.log.info( "Waiting " + main.params['WaitTime'] + " seconds.... " )
+            time.sleep( int(main.params['WaitTime']) )
+
+        main.case("Killing remote ping processes ")
+        result =  result & main.Mininet4.pingKill()
+        utilities.assert_equals(expect=main.TRUE,actual=result)
+
+    def CASE11(self,main) :
+        '''
+        Processes all of the ping data and outputs raw data and an overall average
+        '''
+        import os
+        import time
+        main.log.report("Process ping data")
+        main.case("Processing Ping data")
+        time.sleep(3)
+        #result=os.popen("/home/admin/tools/shell.sh " + main.params['Iterations']).read()
+        result=os.popen("/home/admin/get_reroute_times.py").read()
+        average=result.split(":")[1]
+        main.log.info( "Reroute times are... " )
+        main.log.report( result )
+        try:
+            if float(average) < float(main.params['TargetTime']) :
+                test=main.TRUE
+            else:
+                test=main.FALSE
+        except ValueError:
             main.log.error("Data is corrupted")
             test=main.FALSE
         utilities.assert_equals(expect=main.TRUE,actual=test,onpass="Average is less then the target time!",onfail="Average is worse then target time... ")