Merge pull request #39 from opennetworkinglab/HA

Ha
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 6d999e4..84c4b7f 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1388,6 +1388,9 @@
             
             if num_mult:
                 cmd += " " + str(num_mult)
+                #If app id is specified, then num_mult 
+                #must exist because of the way this command
+                #takes in arguments
                 if app_id:
                     cmd += " " + str(app_id)
             
@@ -1414,7 +1417,7 @@
                     #Append the first result of second parse
                     lat_result.append(result[1].split(" ")[0])
 
-                print lat_result 
+                main.log.info(lat_result) 
                 return lat_result 
             else:
                 return main.TRUE
diff --git a/TestON/tests/IntentPerfNext/IntentPerfNext.params b/TestON/tests/IntentPerfNext/IntentPerfNext.params
index cdf0766..edba27b 100644
--- a/TestON/tests/IntentPerfNext/IntentPerfNext.params
+++ b/TestON/tests/IntentPerfNext/IntentPerfNext.params
@@ -1,5 +1,5 @@
 <PARAMS>
-    <testcases>1,5,2,3</testcases>
+    <testcases>1,5,7,2,3</testcases>
 
     <ENV>
         <cellName>intent_perf_test</cellName>
diff --git a/TestON/tests/IntentPerfNext/IntentPerfNext.py b/TestON/tests/IntentPerfNext/IntentPerfNext.py
index c212e9e..5c8bb2f 100644
--- a/TestON/tests/IntentPerfNext/IntentPerfNext.py
+++ b/TestON/tests/IntentPerfNext/IntentPerfNext.py
@@ -238,9 +238,9 @@
         intent_add_lat_std = \
             round(numpy.std(intent_add_lat_list),1)
         #END ITERATION FOR LOOP
-        main.log.report("Single intent add latency - \n"+
-                "Avg: "+str(intent_add_lat_avg)+" ms\n"+
-                "Std Deviation: "+str(intent_add_lat_std)+" ms")
+        main.log.report("Single intent add latency - ")
+        main.log.report("Avg: "+str(intent_add_lat_avg)+" ms")
+        main.log.report("Std Deviation: "+str(intent_add_lat_std)+" ms")
 
     def CASE3(self, main):
         '''
@@ -281,8 +281,6 @@
 
         intent_reroute_lat_list = []
 
-        print device_id_list
-
         for i in range(0, int(num_iter)):
             #add_point_intent(ingr_device, ingr_port, 
             #                 egr_device, egr_port)
@@ -308,7 +306,7 @@
             #NOTE: this interface is specific to
             #      topo-intentFlower.py topology
             #      reroute case.
-            main.log.info("Disabling interface s2-eth3 <--> s4")
+            main.log.info("Disabling interface s2-eth3")
             main.Mininet1.handle.sendline(
                     "sh ifconfig s2-eth3 down")
             t0_system = time.time()*1000
@@ -355,16 +353,7 @@
                 main.log.info("Intent reroute latency exceeded "+
                         "threshold. Skipping iteration "+str(i))
 
-            #TODO: Possibly put this in the driver function
             main.log.info("Removing intents for next iteration")
-            
-            #NOTE: TODO: Currently, the remove intent will
-            #            not trigger the intent request 
-            #            timestamp. Thus we cannot use the same
-            #            intent to get the latency over iterations.
-            #            we can 1) install different intents every
-            #            time, or 2) look into state machine and 
-            #            determine what timestsamp to get
             main.ONOS1cli.remove_intent(intent_id)
             
             main.log.info("Bringing Mininet interface up for next "+
@@ -377,11 +366,150 @@
         intent_reroute_lat_std = \
             round(numpy.std(intent_reroute_lat_list),1)
         #END ITERATION FOR LOOP
-        main.log.report("Single intent reroute latency - \n"+
-                "Avg: "+str(intent_reroute_lat_avg)+" ms\n"+
-                "Std Deviation: "+str(intent_reroute_lat_std)+" ms")
+        main.log.report("Single intent reroute latency - ")
+        main.log.report("Avg: "+str(intent_reroute_lat_avg)+" ms")
+        main.log.report("Std Deviation: "+str(intent_reroute_lat_std)+" ms")
+        
+    def CASE7(self, main):
+        '''
+        Batch intent reroute latency
+        '''
+        import time
+        import json
+        import requests
+        import os
+        import numpy
+
+        ONOS_ip_list = []
+        for i in range(1, 8):
+            ONOS_ip_list.append(main.params['CTRL']['ip'+str(i)])
+
+        ONOS_user = main.params['CTRL']['user']
+        default_sw_port = main.params['CTRL']['port1']
+    
+        batch_intent_size = main.params['TEST']['batchIntentSize']
+        batch_thresh_min = int(main.params['TEST']['batchThresholdMin'])
+        batch_thresh_max = int(main.params['TEST']['batchThresholdMax'])
+        install_time = main.params['JSON']['installedTime']
+
+        #number of iterations of case
+        num_iter = main.params['TEST']['numIter']
+        num_ignore = int(main.params['TEST']['numIgnore'])
+        num_switch = int(main.params['TEST']['numSwitch'])
+        n_thread = main.params['TEST']['numMult']
+
+        main.log.report("Batch intent installation test of "+
+               batch_intent_size +" intents")
+
+        batch_result_list = []
+
+        #Assign 'linear' switch format for basic intent testing
+        main.Mininet1.assign_sw_controller(
+                sw="1", ip1=ONOS1_ip,port1=default_sw_port)
+        main.Mininet1.assign_sw_controller(
+                sw="2", ip1=ONOS2_ip,port1=default_sw_port)
+        main.Mininet1.assign_sw_controller(
+                sw="3", ip1=ONOS2_ip,port1=default_sw_port)
+        main.Mininet1.assign_sw_controller(
+                sw="4", ip1=ONOS2_ip,port1=default_sw_port)
+        main.Mininet1.assign_sw_controller(
+                sw="5", ip1=ONOS3_ip,port1=default_sw_port)
+
+        time.sleep(10)
+
+        main.log.info("Getting list of available devices")
+        device_id_list = []
+        json_str = main.ONOS1cli.devices()
+        json_obj = json.loads(json_str)
+        for device in json_obj:
+            device_id_list.append(device['id'])
+
+        batch_install_lat = []
+        batch_withdraw_lat = []
+        sleep_time = 10
+        
+        base_dir = "/tmp/"
+        max_install_lat = []
+
+        for i in range(0, int(num_iter)):
+            main.log.info("Pushing "+
+                    str(int(batch_intent_size)*int(n_thread))+
+                    " intents. Iteration "+str(i))
             
+            main.ONOS1cli.push_test_intents(
+                "of:0000000000000001/1",
+                "of:0000000000000005/1",
+                1000, num_mult="1", app_id="1")
+               
+            #TODO: Check for installation success then proceed
+            time.sleep(30)
+            
+            #NOTE: this interface is specific to
+            #      topo-intentFlower.py topology
+            #      reroute case.
+            main.log.info("Disabling interface s2-eth3")
+            main.Mininet1.handle.sendline(
+                    "sh ifconfig s2-eth3 down")
+            t0_system = time.time()*1000
+
+            #TODO: Wait sufficient time for intents to install
+            time.sleep(10)
+
+            #TODO: get intent installation time
+            
+            #Obtain metrics from ONOS 1, 2, 3
+            intents_json_str_1 = main.ONOS1cli.intents_events_metrics()
+            intents_json_str_2 = main.ONOS2cli.intents_events_metrics()
+            intents_json_str_3 = main.ONOS3cli.intents_events_metrics()
+
+            intents_json_obj_1 = json.loads(intents_json_str_1)
+            intents_json_obj_2 = json.loads(intents_json_str_2)
+            intents_json_obj_3 = json.loads(intents_json_str_3)
+
+            #Parse values from the json object
+            intent_install_1 = \
+                    intents_json_obj_1[install_time]['value']
+            intent_install_2 = \
+                    intents_json_obj_2[install_time]['value']
+            intent_install_3 = \
+                    intents_json_obj_3[install_time]['value']
+
+            intent_reroute_lat_1 = \
+                    int(intent_install_1) - int(t0_system)
+            intent_reroute_lat_2 = \
+                    int(intent_install_2) - int(t0_system)
+            intent_reroute_lat_3 = \
+                    int(intent_install_3) - int(t0_system)
+            
+            intent_reroute_lat_avg = \
+                    (intent_reroute_lat_1 + 
+                     intent_reroute_lat_2 +
+                     intent_reroute_lat_3 ) / 3
+    
+            main.log.info("Intent reroute latency avg for iteration "+
+                    str(i)+": "+str(intent_reroute_lat_avg))
+            #TODO: Remove intents for next iteration
+            
+            time.sleep(5)
+
+            intents_str = main.ONOS1cli.intents()
+            intents_json = json.loads(intents_str)
+            for intents in intents_json:
+                intent_id = intents['id']
+                if intent_id:
+                    main.ONOS1cli.remove_intent(intent_id)
+
+            main.Mininet1.handle.sendline(
+                    "sh ifconfig s2-eth3 up")
+            
+            main.log.info("Intents removed and port back up")
+
+
     def CASE4(self, main):
+        '''
+        Batch intent install
+        '''
+        
         import time
         import json
         import requests
@@ -596,7 +724,7 @@
 
         global cluster_count
         cluster_count += 2
-        main.log.info("Increasing cluster size to "+
+        main.log.report("Increasing cluster size to "+
                 str(cluster_count))
 
         install_result = main.FALSE
@@ -701,7 +829,7 @@
             main.ONOSbench.push_test_intents_shell(
                 arg1,
                 arg2, 
-                150, "/tmp/temp.txt", "10.128.174.1",
+                100, "/tmp/temp.txt", "10.128.174.1",
                 num_mult="10", app_id=appid,report=False)
             #main.ONOSbench.push_test_intents_shell(
             #    "of:0000000000001002/1",
@@ -714,5 +842,5 @@
             #    133, "/tmp/temp3.txt", "10.128.174.3",
             #    num_mult="6", app_id="3",report=False)
    
-            time.sleep(0.1)
+            time.sleep(0.2)
 
diff --git a/TestON/tests/ProdFunc/ProdFunc.params b/TestON/tests/ProdFunc/ProdFunc.params
index 6879c37..5de8118 100755
--- a/TestON/tests/ProdFunc/ProdFunc.params
+++ b/TestON/tests/ProdFunc/ProdFunc.params
@@ -1,6 +1,6 @@
 <PARAMS>
     
-    <testcases>1,4,10,5,6,7,8,6,8,9</testcases>
+    <testcases>1,4,10,5,6,7,8,9,2,8,20,21,22,10,23,24</testcases>
 
     #Environment variables
     <ENV>
@@ -37,7 +37,13 @@
 
     <timers>
         <LinkDiscovery>5</LinkDiscovery>
-        <SwitchDiscovery>31</SwitchDiscovery>
+        <SwitchDiscovery>15</SwitchDiscovery>
     </timers>
 
+    <OPTICAL>
+        <jsonfile> /home/admin/ONOS/tools/test/topos/oe-nonlinear-4.json </jsonfile>
+    </OPTICAL>    
+
+
+
 </PARAMS>
diff --git a/TestON/tests/ProdFunc/ProdFunc.py b/TestON/tests/ProdFunc/ProdFunc.py
index 05c420e..b786e8b 100755
--- a/TestON/tests/ProdFunc/ProdFunc.py
+++ b/TestON/tests/ProdFunc/ProdFunc.py
@@ -44,13 +44,17 @@
         main.step("Git checkout and pull master and get version")
         main.ONOSbench.git_checkout("master")
         git_pull_result = main.ONOSbench.git_pull()
-        print "git_pull_result = ", git_pull_result
+        main.log.info("git_pull_result = " +git_pull_result)
         version_result = main.ONOSbench.get_version(report=True)
     
         if git_pull_result == 1:
             main.step("Using mvn clean & install")
             clean_install_result = main.ONOSbench.clean_install()
             #clean_install_result = main.TRUE
+        elif git_pull_result == 0:
+            main.log.report("Git Pull Failed, look into logs for detailed reason")
+            main.cleanup()
+            main.exit() 
          
         main.step("Creating ONOS package")
         package_result = main.ONOSbench.onos_package()
@@ -81,6 +85,42 @@
                 onpass="Test startup successful",
                 onfail="Test startup NOT successful")
 
+    def CASE2(self, main) :
+        '''  
+        Switch Down
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time 
+        import json
+ 
+        main.case("Switch down discovery")
+        main.log.report("This testcase is testing a switch down discovery")
+        main.log.report("__________________________________")
+
+        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+        description = "Killing a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        #TODO: Make this switch parameterizable
+        main.step("Kill s28 ")
+        main.log.report("Deleting s28")
+        #FIXME: use new dynamic topo functions
+        main.Mininet1.del_switch("s28")
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the deleted switch
+        device = main.ONOS2.get_device(dpid="0028")
+        print "device = ", device
+        if device[u'available'] == 'False':
+            case2_result = main.FALSE
+        else:
+            case2_result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE, actual=case2_result,
+                onpass="Switch down discovery successful",
+                onfail="Switch down discovery failed")
+
     def CASE11(self, main):
         '''
         Cleanup sequence:
@@ -247,8 +287,9 @@
         print "links_result = ", links_result
         print "_________________________________"
         
-
-
+        #NOTE:Since only point intents are added, there is no requirement to discover the hosts
+                #Therfore, the below portion of the code is commented.
+        '''
         #Discover hosts using pingall
         pingall_result = main.LincOE2.pingall()    
     
@@ -269,8 +310,9 @@
             print "Number of hosts = %d and is wrong" %hostCount
             main.log.info("Number of hosts = " + str(hostCount) +" and is wrong")
             hostDiscovery = main.FALSE
-        
-        case22_result = opticalSW_result and packetSW_result and hostDiscovery
+        '''
+
+        case22_result = opticalSW_result and packetSW_result
         utilities.assert_equals(expect=main.TRUE, actual=case22_result,
                 onpass="Packet optical topology discovery successful",
                 onfail="Packet optical topology discovery failed")
@@ -286,12 +328,12 @@
         main.step("Adding point intents")
         ptp_intent_result = main.ONOS3.add_point_intent("of:0000ffffffff0001/1", "of:0000ffffffff0002/1")
         if ptp_intent_result == main.TRUE:
-            get_intent_result = main.ONOS3.intents()
+            get_intent_result = main.ONOS3.intents(json_format = False)
             main.log.info("Point to point intent install successful")
 
         ptp_intent_result = main.ONOS3.add_point_intent("of:0000ffffffff0002/1", "of:0000ffffffff0001/1")
         if ptp_intent_result == main.TRUE:
-            get_intent_result = main.ONOS3.intents()
+            get_intent_result = main.ONOS3.intents(json_format = False)
             main.log.info("Point to point intent install successful")
 
         time.sleep(10)
@@ -300,7 +342,7 @@
 
         # Sleep for 30 seconds to provide time for the intent state to change
         time.sleep(30)
-        intentHandle = main.ONOS3.intents()        
+        intentHandle = main.ONOS3.intents(json_format = False)        
         main.log.info("intents :" + intentHandle)        
  
         Ping_Result = main.TRUE
@@ -313,8 +355,6 @@
             Ping_Result = main.FALSE
             main.log.info("Ping between h1 and h2  failed. Making attempt number "+str(count) + " in 2 seconds")
             time.sleep(2)
-            ping = main.LincOE2.pingHostOptical(src="h1", target="h2")
-            #ping = main.LincOE2.pinghost()
         elif ping==main.FALSE:
             main.log.info("All ping attempts between h1 and h2 have failed")
             Ping_Result = main.FALSE
@@ -345,31 +385,39 @@
             and do a ping test. If rerouting is successful, ping should pass. also check the flows
         '''
         main.log.report("This testcase tests rerouting and pings mininet hosts")
-        main.step("Test rerouting and pings mininet hosts")
-        main.case("Bring a port down and verify the link state")
+        main.case("Test rerouting and pings mininet hosts")
+        main.step("Bring a port down and verify the link state")
         main.LincOE1.port_down(sw_id="1", pt_id="22") 
-        links = main.ONOS3.links()
-        main.log.info(main.ONOS3.links)
+        links_nonjson = main.ONOS3.links(json_format = False)
+        main.log.info("links = " +links_nonjson)
 
+        links = main.ONOS3.links()
+        main.log.info("links = " +links)
+        
         links_result = json.loads(links)
         links_state_result = main.FALSE
         for item in links_result:
-            if item['src'] == "of:0000ffffffffff01/22" and item['dst'] == "of:0000ffffffffff04/30":
-                links_state = item['state']
-                if links_state == "INACTIVE":
-                    main.log.info("Links state is inactive as expected due to one of the ports being down")
-                    main.log.report("Links state is inactive as expected due to one of the ports being down")
-                    links_state_result = main.TRUE
-                    break
-                else:
-                    main.log.info("Links state is not inactive as expected")
-                    main.log.report("Links state is not inactive as expected")
-                    links_state_result = main.FALSE
+            if item['src']['device'] == "of:0000ffffffffff01" and item['src']['port'] == "22":
+                if item['dst']['device'] == "of:0000ffffffffff04" and item['dst']['port'] == "30":
+                    links_state = item['state']
+                    if links_state == "INACTIVE":
+                        main.log.info("Links state is inactive as expected due to one of the ports being down")
+                        main.log.report("Links state is inactive as expected due to one of the ports being down")
+                        links_state_result = main.TRUE
+                        break
+                    else:
+                        main.log.info("Links state is not inactive as expected")
+                        main.log.report("Links state is not inactive as expected")
+                        links_state_result = main.FALSE
 
         print "links_state_result = ", links_state_result
-        main.case("Verify Rerouting by a ping test")
+        time.sleep(10)
+        flowHandle = main.ONOS3.flows()
+        main.log.info("flows :" + flowHandle)
+
+        main.step("Verify Rerouting by a ping test")
         Ping_Result = main.TRUE
-        count = 1
+        count = 1        
         main.log.info("\n\nh1 is Pinging h2")
         ping = main.LincOE2.pingHostOptical(src="h1", target="h2")
         #ping = main.LincOE2.pinghost()
@@ -378,8 +426,6 @@
             Ping_Result = main.FALSE
             main.log.info("Ping between h1 and h2  failed. Making attempt number "+str(count) + " in 2 seconds")
             time.sleep(2)
-            ping = main.LincOE2.pingHostOptical(src="h1", target="h2")
-            #ping = main.LincOE2.pinghost()
         elif ping==main.FALSE:
             main.log.info("All ping attempts between h1 and h2 have failed")
             Ping_Result = main.FALSE
@@ -390,9 +436,9 @@
             main.log.info("Unknown error")
             Ping_Result = main.ERROR
 
-        if Ping_Result==main.FALSE:
-            main.log.report("Ping test successful ")
         if Ping_Result==main.TRUE:
+            main.log.report("Ping test successful ")
+        if Ping_Result==main.FALSE:
             main.log.report("Ping test failed")
 
         case24_result = Ping_Result and links_state_result
diff --git a/TestON/tests/ProdFunc/ProdFunc.topo b/TestON/tests/ProdFunc/ProdFunc.topo
index 5d453a6..7dff686 100755
--- a/TestON/tests/ProdFunc/ProdFunc.topo
+++ b/TestON/tests/ProdFunc/ProdFunc.topo
@@ -28,12 +28,22 @@
             <COMPONENTS> </COMPONENTS>
         </ONOS2>
 
+         <ONOS3>
+            <host>10.128.10.11</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>OnosCliDriver</type>
+            <connect_order>4</connect_order>
+            <COMPONENTS> </COMPONENTS>
+        </ONOS3>
+       
+
         <Mininet1>
             <host>10.128.10.11</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>MininetCliDriver</type>
-            <connect_order>4</connect_order>
+            <connect_order>5</connect_order>
             <COMPONENTS>
                 #Specify the Option for mininet
                 <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
@@ -48,14 +58,46 @@
             <user>admin</user>
             <password>onos_test</password>
             <type>RemoteMininetDriver</type>
-            <connect_order>5</connect_order>
+            <connect_order>6</connect_order>
             <COMPONENTS>
                 #Specify the Option for mininet
                 <arg1> --custom ~/mininet/custom/topo-HA.py </arg1>
                 <arg2> --topo mytopo </arg2>
-                <arg3> --switch ovs,protocols=OpenFlow10 </arg3>
+                <arg3> --switch ovs,protocols=OpenFlow13 </arg3>
                 <controller> remote </controller>
             </COMPONENTS>
         </Mininet2>
+
+        <LincOE1>
+            <host>10.128.20.30</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>LincOEDriver</type>
+            <connect_order>7</connect_order>
+            <COMPONENTS>
+                <arg1> </arg1>
+            </COMPONENTS>
+        </LincOE1>
+
+        <LincOE2>
+            <host>10.128.20.30</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>RemoteMininetDriver</type>
+            <connect_order>8</connect_order>
+            <COMPONENTS>
+                <arg1> sudo python /home/admin/optical.py </arg1>
+                <arg2> </arg2>
+            </COMPONENTS>
+        </LincOE2>
+
+        <LincOE3>
+            <host>10.128.20.30</host>
+            <user>admin</user>
+            <password>onos_test</password>
+            <type>LincOEDriver</type>
+            <connect_order>9</connect_order>
+        </LincOE3>
+ 
     </COMPONENT>
 </TOPOLOGY>
diff --git a/TestON/tests/ProdFunc13/ProdFunc13.params b/TestON/tests/ProdFunc13/ProdFunc13.params
index 2af3237..bb3c11c 100755
--- a/TestON/tests/ProdFunc13/ProdFunc13.params
+++ b/TestON/tests/ProdFunc13/ProdFunc13.params
@@ -1,6 +1,6 @@
 <PARAMS>
     
-    <testcases>1,4,10,5,6,7,8,9,20,21,22,10,23,24</testcases>
+    <testcases>1,4,10,5,6,7,8,9,2,20,21,22,10,23,24</testcases>
 
     #Environment variables
     <ENV>
@@ -37,7 +37,7 @@
 
     <timers>
         <LinkDiscovery>5</LinkDiscovery>
-        <SwitchDiscovery>31</SwitchDiscovery>
+        <SwitchDiscovery>15</SwitchDiscovery>
     </timers>
 
     <OPTICAL>
diff --git a/TestON/tests/ProdFunc13/ProdFunc13.py b/TestON/tests/ProdFunc13/ProdFunc13.py
index abb99d2..babf853 100755
--- a/TestON/tests/ProdFunc13/ProdFunc13.py
+++ b/TestON/tests/ProdFunc13/ProdFunc13.py
@@ -44,13 +44,17 @@
         main.step("Git checkout and pull master and get version")
         main.ONOSbench.git_checkout("master")
         git_pull_result = main.ONOSbench.git_pull()
-        print "git_pull_result = ", git_pull_result
+        main.log.info("git_pull_result = " +git_pull_result)
         version_result = main.ONOSbench.get_version(report=True)
     
         if git_pull_result == 1:
             main.step("Using mvn clean & install")
             clean_install_result = main.ONOSbench.clean_install()
             #clean_install_result = main.TRUE
+        elif git_pull_result == 0:
+            main.log.report("Git Pull Failed, look into logs for detailed reason")
+            main.cleanup()
+            main.exit() 
          
         main.step("Creating ONOS package")
         package_result = main.ONOSbench.onos_package()
@@ -81,6 +85,42 @@
                 onpass="Test startup successful",
                 onfail="Test startup NOT successful")
 
+    def CASE2(self, main) :
+        '''  
+        Switch Down
+        '''
+        #NOTE: You should probably run a topology check after this
+        import time 
+        import json
+ 
+        main.case("Switch down discovery")
+        main.log.report("This testcase is testing a switch down discovery")
+        main.log.report("__________________________________")
+
+        switch_sleep = int(main.params['timers']['SwitchDiscovery'])
+
+        description = "Killing a switch to ensure it is discovered correctly"
+        main.log.report(description)
+        main.case(description)
+
+        #TODO: Make this switch parameterizable
+        main.step("Kill s28 ")
+        main.log.report("Deleting s28")
+        #FIXME: use new dynamic topo functions
+        main.Mininet1.del_switch("s28")
+        main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered")
+        time.sleep(switch_sleep)
+        #Peek at the deleted switch
+        device = main.ONOS2.get_device(dpid="0028")
+        print "device = ", device
+        if device[u'available'] == 'False':
+            case2_result = main.FALSE
+        else:
+            case2_result = main.TRUE
+        utilities.assert_equals(expect=main.TRUE, actual=case2_result,
+                onpass="Switch down discovery successful",
+                onfail="Switch down discovery failed")
+
     def CASE11(self, main):
         '''
         Cleanup sequence:
@@ -315,8 +355,6 @@
             Ping_Result = main.FALSE
             main.log.info("Ping between h1 and h2  failed. Making attempt number "+str(count) + " in 2 seconds")
             time.sleep(2)
-            ping = main.LincOE2.pingHostOptical(src="h1", target="h2")
-            #ping = main.LincOE2.pinghost()
         elif ping==main.FALSE:
             main.log.info("All ping attempts between h1 and h2 have failed")
             Ping_Result = main.FALSE
@@ -388,8 +426,6 @@
             Ping_Result = main.FALSE
             main.log.info("Ping between h1 and h2  failed. Making attempt number "+str(count) + " in 2 seconds")
             time.sleep(2)
-            ping = main.LincOE2.pingHostOptical(src="h1", target="h2")
-            #ping = main.LincOE2.pinghost()
         elif ping==main.FALSE:
             main.log.info("All ping attempts between h1 and h2 have failed")
             Ping_Result = main.FALSE