Merge branch 'master' of https://github.com/OPENNETWORKINGLAB/ONLabTest
diff --git a/TestON/tests/RCOnosPerf4nodes/RCOnosPerf4nodes.py b/TestON/tests/RCOnosPerf4nodes/RCOnosPerf4nodes.py
index 90fd3a8..b4b73f4 100644
--- a/TestON/tests/RCOnosPerf4nodes/RCOnosPerf4nodes.py
+++ b/TestON/tests/RCOnosPerf4nodes/RCOnosPerf4nodes.py
@@ -34,6 +34,7 @@
         main.Zookeeper2.start()
         main.Zookeeper3.start()
         main.Zookeeper4.start()
+        time.sleep(4)
         main.RamCloud1.del_db()
         main.RamCloud2.del_db()
         main.RamCloud3.del_db()
@@ -440,6 +441,7 @@
 
     def CASE66(self, main):
         main.log.report("Checking ONOS logs for exceptions")
+        count = 0
         check1 = main.ONOS1.check_exceptions()
         main.log.report("Exceptions in ONOS1 logs: \n" + check1)
         check2 = main.ONOS2.check_exceptions()
@@ -448,9 +450,10 @@
         main.log.report("Exceptions in ONOS3 logs: \n" + check3)
         check4 = main.ONOS4.check_exceptions()
         main.log.report("Exceptions in ONOS4 logs: \n" + check4)
-        result = main.FALSE
+        result = main.TRUE
         if (check1 or check2 or check3 or check4):
-            result = main.TRUE
-        utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail="Exceptions found")
+            result = main.FALSE
+            count = len(check1.splitlines()) + len(check2.splitlines()) + len(check3.splitlines()) + len(check4.splitlines())
+        utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail=str(count) + " Exceptions were found in the logs")
 
 
diff --git a/TestON/tests/RCOnosSanity4nodesJ/RCOnosSanity4nodesJ.py b/TestON/tests/RCOnosSanity4nodesJ/RCOnosSanity4nodesJ.py
index ce4dde8..e7a5fa0 100644
--- a/TestON/tests/RCOnosSanity4nodesJ/RCOnosSanity4nodesJ.py
+++ b/TestON/tests/RCOnosSanity4nodesJ/RCOnosSanity4nodesJ.py
@@ -436,7 +436,6 @@
         main.log.report("Check initially hostMAC/IP exist on the mininet...")
         host = main.params['YANK']['hostname']
         mac = main.params['YANK']['hostmac']
-        hostip = main.params['YANK']['hostip']
         RestIP1 = main.params['RESTCALL']['restIP1']
         RestPort = main.params['RESTCALL']['restPort']
         url = main.params['RESTCALL']['restURL']
@@ -450,16 +449,15 @@
         main.log.info("\n\t\t\t\t ping issue one ping from " + str(host) + " to generate arp to switch. Ping result is not important" )
         ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254")
         time.sleep(t_restwait)
-        Reststatus, Switch, Port, MAC = main.ONOS1.find_host(RestIP1,RestPort,url, hostip)
-        main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus))
+        Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac)
+        main.log.report("Number of host with MAC address = " + mac + " found by ONOS is: " + str(Reststatus))
         if Reststatus == 1:
-            main.log.report("\t PASSED - Found host IP = " + hostip + "; MAC = " + "".join(MAC) + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + "".join(Port))
+            main.log.report("\t PASSED - Found host mac = " + mac + ";  attached to switchDPID = " +"".join(Switch) + "; at port = " + str(Port[0]))
             result1 = main.TRUE
         elif Reststatus > 1:
-            main.log.report("\t FAILED - Host " + host + " with MAC:" + mac + " has " + str(Reststatus) + " duplicated IP addresses. FAILED")
+            main.log.report("\t FAILED - Host " + host + " with MAC:" + mac + " has " + str(Reststatus) + " duplicated mac  addresses. FAILED")
             main.log.report("switches are: " + "; ".join(Switch))
             main.log.report("Ports are: " + "; ".join(Port))
-            main.log.report("MACs are: " + "; ".join(MAC))
             result1 = main.FALSE
         elif Reststatus == 0 and Switch == []:
             main.log.report("\t FAILED - Host " + host + " with MAC:" + mac + " does not exist. FAILED")
@@ -480,11 +478,11 @@
         main.log.info("\n\t\t\t\t ping issue one ping from " + str(host) + " to generate arp to switch. Ping result is not important" )
         ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254")
         time.sleep(t_restwait)
-        Reststatus, Switch, Port, MAC = main.ONOS1.find_host(RestIP1,RestPort,url, hostip)
+        Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac)
 
-        main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus))
+        main.log.report("Number of host with MAC = " + mac + " found by ONOS is: " + str(Reststatus))
         if Reststatus == 1:
-            main.log.report("\tFAILED - Found host IP = " + hostip + "; MAC = " + "".join(MAC) + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + "".join(Port))
+            main.log.report("\tFAILED - Found host MAC = " + mac + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + str(Port))
             result2 = main.FALSE
         elif Reststatus > 1:
             main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " has " + str(Reststatus) + " duplicated IP addresses. FAILED")
@@ -509,11 +507,11 @@
 
         ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254")
         time.sleep(t_restwait)
-        Reststatus, Switch, Port, MAC = main.ONOS1.find_host(RestIP1,RestPort,url, hostip)
+        Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac)
 
-        main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus))
+        main.log.report("Number of host with MAC " + mac + " found by ONOS is: " + str(Reststatus))
         if Reststatus == 1:
-            main.log.report("\tPASSED - Found host IP = " + hostip + "; MAC = " + "".join(MAC) + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + "".join(Port))
+            main.log.report("\tPASSED - Found host MAC = " + mac + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + str(Port[0]))
             result3 = main.TRUE
         elif Reststatus > 1:
             main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " has " + str(Reststatus) + " duplicated IP addresses. FAILED")
@@ -539,11 +537,11 @@
 
         ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254")
         time.sleep(t_restwait)
-        Reststatus, Switch, Port, MAC = main.ONOS1.find_host(RestIP1,RestPort,url, hostip)
+        Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac)
 
         main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus))
         if Reststatus == 1:
-            main.log.report("\tPASSED - Found host IP = " + hostip + "; MAC = " + "".join(MAC) + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + "".join(Port))
+            main.log.report("\tPASSED - Found host MAC = " + mac + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + str(Port[0]))
             result4 = main.TRUE
         elif Reststatus > 1:
             main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " has " + str(Reststatuas) + " duplicated IP addresses. FAILED")
@@ -557,6 +555,15 @@
         else:# check if rest server is working
             main.log.error("Issue with find host")
             result4 = main.FALSE
+        time.sleep(6)
+        Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url,mac)
+        main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus))
+        if Reststatus ==1:
+            main.log.report("\t FAILED - Host " + host + "with MAC:" + str(mac) + "was still found after expected timeout")
+        elif Reststatus>1:
+            main.log.report("\t FAILED - Host " + host + "with MAC:" + str(mac) + "was still found after expected timeout(multiple found)")
+        elif Reststatus==0:
+            main.log.report("\t PASSED - Device cleared after timeout")
 
         result = result1 and result2 and result3 and result4
         utilities.assert_equals(expect=main.TRUE,actual=result,onpass="DEVICE DISCOVERY TEST PASSED PLUG/UNPLUG/MOVE TEST",onfail="DEVICE DISCOVERY TEST FAILED")
@@ -602,10 +609,10 @@
         main.log.report("Exceptions in ONOS3 logs: \n" + check3)
         check4 = main.ONOS4.check_exceptions()
         main.log.report("Exceptions in ONOS4 logs: \n" + check4)
-        result = main.FALSE
+        result = main.TRUE
         if (check1 or check2 or check3 or check4):
-            result = main.TRUE
+            result = main.FALSE
             count = len(check1.splitlines()) + len(check2.splitlines()) + len(check3.splitlines()) + len(check4.splitlines())
-        utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail=str(count) + "Exceptions were found in the logs")
+        utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail=str(count) + " Exceptions were found in the logs")
 
 
diff --git a/TestON/tests/RCOnosScale4nodes/RCOnosScale4nodes.py b/TestON/tests/RCOnosScale4nodes/RCOnosScale4nodes.py
index 53bc154..9e6cb7b 100644
--- a/TestON/tests/RCOnosScale4nodes/RCOnosScale4nodes.py
+++ b/TestON/tests/RCOnosScale4nodes/RCOnosScale4nodes.py
@@ -21,6 +21,10 @@
         main.RamCloud2.stop_serv()
         main.RamCloud3.stop_serv()
         main.RamCloud4.stop_serv()
+        main.Zookeeper1.start()
+        main.Zookeeper2.start()
+        main.Zookeeper3.start()
+        main.Zookeeper4.start()
         time.sleep(4)
         main.RamCloud1.del_db()
         main.RamCloud2.del_db()
@@ -273,8 +277,9 @@
  
         utilities.assert_equals(expect=main.TRUE,actual=test)
 
-      def CASE66(self, main):
+    def CASE66(self, main):
         main.log.report("Checking ONOS logs for exceptions")
+        count = 0
         check1 = main.ONOS1.check_exceptions()
         main.log.report("Exceptions in ONOS1 logs: \n" + check1)
         check2 = main.ONOS2.check_exceptions()
@@ -283,9 +288,10 @@
         main.log.report("Exceptions in ONOS3 logs: \n" + check3)
         check4 = main.ONOS4.check_exceptions()
         main.log.report("Exceptions in ONOS4 logs: \n" + check4)
-        result = main.FALSE
+        result = main.TRUE
         if (check1 or check2 or check3 or check4):
-            result = main.TRUE
-        utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail="Exceptions found")
+            result = main.FALSE
+            count = len(check1.splitlines()) + len(check2.splitlines()) + len(check3.splitlines()) + len(check4.splitlines())
+        utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail=str(count) + " Exceptions were found in the logs")
 
- 
+
diff --git a/TestON/tests/RRCOnosSanity4nodesJ/RRCOnosSanity4nodesJ.params b/TestON/tests/RRCOnosSanity4nodesJ/RRCOnosSanity4nodesJ.params
index 3f0c13a..c67397f 100644
--- a/TestON/tests/RRCOnosSanity4nodesJ/RRCOnosSanity4nodesJ.params
+++ b/TestON/tests/RRCOnosSanity4nodesJ/RRCOnosSanity4nodesJ.params
@@ -44,6 +44,6 @@
 	<restIP1>10.128.4.151</restIP1>
 	<restIP2>10.128.4.152</restIP2>
 	<restPort>8080</restPort>
-	<restURL>/wm/onos/topology/switches/json</restURL>
+	<restURL>/wm/onos/topology/devices/json</restURL>
     </RESTCALL>
 </PARAMS>      
diff --git a/TestON/tests/RRCOnosSanity4nodesJ/RRCOnosSanity4nodesJ.py b/TestON/tests/RRCOnosSanity4nodesJ/RRCOnosSanity4nodesJ.py
index a5af4a4..5f1db04 100644
--- a/TestON/tests/RRCOnosSanity4nodesJ/RRCOnosSanity4nodesJ.py
+++ b/TestON/tests/RRCOnosSanity4nodesJ/RRCOnosSanity4nodesJ.py
@@ -103,10 +103,10 @@
         data = main.ONOS1.isup()
         if data == main.FALSE: 
             main.log.report("Something is funny... restarting ONOS")
-            main.ONOS1.stop()
+        #    main.ONOS1.stop()
             time.sleep(3)
-            main.ONOS1.start()
-            time.sleep(5) 
+        #    main.ONOS1.start()
+        #    time.sleep(5) 
             data = main.ONOS1.isup()
         utilities.assert_equals(expect=main.TRUE,actual=data,onpass="ONOS is up and running!",onfail="ONOS didn't start...")
            
@@ -434,12 +434,11 @@
         main.log.report("Check initially hostMAC/IP exist on the mininet...")
         host = main.params['YANK']['hostname']
         mac = main.params['YANK']['hostmac']
-        hostip = main.params['YANK']['hostip']
         RestIP1 = main.params['RESTCALL']['restIP1']
         RestPort = main.params['RESTCALL']['restPort']
         url = main.params['RESTCALL']['restURL']
-       
-        t_topowait = 0
+    
+        t_topowait = 0 
         t_restwait = 10
         main.log.report( "Wait time from topo change to ping set to " + str(t_topowait))
         main.log.report( "Wait time from ping to rest call set to " + str(t_restwait))
@@ -448,16 +447,15 @@
         main.log.info("\n\t\t\t\t ping issue one ping from " + str(host) + " to generate arp to switch. Ping result is not important" )
         ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254")
         time.sleep(t_restwait)
-        Reststatus, Switch, Port, MAC = main.ONOS1.find_host(RestIP1,RestPort,url, hostip)
-        main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus))
+        Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac)
+        main.log.report("Number of host with MAC address = " + mac + " found by ONOS is: " + str(Reststatus))        
         if Reststatus == 1:
-            main.log.report("\t PASSED - Found host IP = " + hostip + "; MAC = " + "".join(MAC) + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + "".join(Port))
-            result1 = main.TRUE
+            main.log.report("\t PASSED - Found host mac = " + mac + ";  attached to switchDPID = " +"".join(Switch) + "; at port = " + str(Port[0]))
+            result1 = main.TRUE 
         elif Reststatus > 1:
-            main.log.report("\t FAILED - Host " + host + " with MAC:" + mac + " has " + str(Reststatus) + " duplicated IP addresses. FAILED")
+            main.log.report("\t FAILED - Host " + host + " with MAC:" + mac + " has " + str(Reststatus) + " duplicated mac  addresses. FAILED")
             main.log.report("switches are: " + "; ".join(Switch))
             main.log.report("Ports are: " + "; ".join(Port))
-            main.log.report("MACs are: " + "; ".join(MAC))
             result1 = main.FALSE
         elif Reststatus == 0 and Switch == []:
             main.log.report("\t FAILED - Host " + host + " with MAC:" + mac + " does not exist. FAILED")
@@ -467,7 +465,7 @@
             result1 = main.FALSE
 
 
-        ##### Step to yank out "s1-eth1" from s1, which is on autoONOS1 #####
+                ##### Step to yank out "s1-eth1" from s1, which is on autoONOS1 #####
 
         main.log.report("Yank out s1-eth1")
         main.case("Yankout s6-eth1 (link to h1) from s1")
@@ -478,12 +476,12 @@
         main.log.info("\n\t\t\t\t ping issue one ping from " + str(host) + " to generate arp to switch. Ping result is not important" )
         ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254")
         time.sleep(t_restwait)
-        Reststatus, Switch, Port, MAC = main.ONOS1.find_host(RestIP1,RestPort,url, hostip)
+        Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac)
 
-        main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus))
+        main.log.report("Number of host with MAC = " + mac + " found by ONOS is: " + str(Reststatus))
         if Reststatus == 1:
-            main.log.report("\tFAILED - Found host IP = " + hostip + "; MAC = " + "".join(MAC) + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + "".join(Port))
-            result2 = main.FALSE
+            main.log.report("\tFAILED - Found host MAC = " + mac + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + str(Port))
+            result2 = main.FALSE 
         elif Reststatus > 1:
             main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " has " + str(Reststatus) + " duplicated IP addresses. FAILED")
             main.log.report("switches are: " + "; ".join(Switch))
@@ -496,7 +494,7 @@
         else:# check if rest server is working
             main.log.error("Issue with find host")
             result2 = main.FALSE
-         
+
         ##### Step to plug "s1-eth1" to s6, which is on autoONOS3  ######
         main.log.report("Plug s1-eth1 into s6")
         main.case("Plug s1-eth1 to s6")
@@ -507,17 +505,17 @@
 
         ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254")
         time.sleep(t_restwait)
-        Reststatus, Switch, Port, MAC = main.ONOS1.find_host(RestIP1,RestPort,url, hostip)
+        Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac)
 
-        main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus))
+        main.log.report("Number of host with MAC " + mac + " found by ONOS is: " + str(Reststatus))
         if Reststatus == 1:
-            main.log.report("\tPASSED - Found host IP = " + hostip + "; MAC = " + "".join(MAC) + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + "".join(Port))
+            main.log.report("\tPASSED - Found host MAC = " + mac + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + str(Port[0]))
             result3 = main.TRUE
         elif Reststatus > 1:
             main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " has " + str(Reststatus) + " duplicated IP addresses. FAILED")
             main.log.report("switches are: " + "; ".join(Switch))
             main.log.report("Ports are: " + "; ".join(Port))
-            main.log.report("MACs are: " + "; ".join(MAC))            
+            main.log.report("MACs are: " + "; ".join(MAC))
             result3 = main.FALSE
         elif Reststatus == 0 and Switch == []:
             main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " does not exist. FAILED")
@@ -525,7 +523,7 @@
         else:# check if rest server is working
             main.log.error("Issue with find host")
             result3 = main.FALSE
-
+       
         ###### Step to put interface "s1-eth1" back to s1"#####
         main.log.report("Move s1-eth1 back on to s1")
         main.case("Move s1-eth1 back to s1")
@@ -537,17 +535,17 @@
 
         ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254")
         time.sleep(t_restwait)
-        Reststatus, Switch, Port, MAC = main.ONOS1.find_host(RestIP1,RestPort,url, hostip)
+        Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac)
 
         main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus))
         if Reststatus == 1:
-            main.log.report("\tPASSED - Found host IP = " + hostip + "; MAC = " + "".join(MAC) + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + "".join(Port))
+            main.log.report("\tPASSED - Found host MAC = " + mac + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + str(Port[0]))
             result4 = main.TRUE
         elif Reststatus > 1:
             main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " has " + str(Reststatuas) + " duplicated IP addresses. FAILED")
             main.log.report("switches are: " + "; ".join(Switch))
             main.log.report("Ports are: " + "; ".join(Port))
-            main.log.report("MACs are: " + "; ".join(MAC))            
+            main.log.report("MACs are: " + "; ".join(MAC))
             result4 = main.FALSE
         elif Reststatus == 0 and Switch == []:
             main.log.report("\t FAILED -Host " + host + " with MAC:" + str(mac) + " does not exist. FAILED")
@@ -559,6 +557,7 @@
         result = result1 and result2 and result3 and result4
         utilities.assert_equals(expect=main.TRUE,actual=result,onpass="DEVICE DISCOVERY TEST PASSED PLUG/UNPLUG/MOVE TEST",onfail="DEVICE DISCOVERY TEST FAILED")
 
+
 # Run a pure ping test. 
 
     def CASE31(self, main):
@@ -588,9 +587,9 @@
             main.log.report("\tPING TEST FAIL")
         utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE")
 
-
     def CASE66(self, main):
         main.log.report("Checking ONOS logs for exceptions")
+        count = 0
         check1 = main.ONOS1.check_exceptions()
         main.log.report("Exceptions in ONOS1 logs: \n" + check1)
         check2 = main.ONOS2.check_exceptions()
@@ -599,9 +598,10 @@
         main.log.report("Exceptions in ONOS3 logs: \n" + check3)
         check4 = main.ONOS4.check_exceptions()
         main.log.report("Exceptions in ONOS4 logs: \n" + check4)
-        result = main.FALSE
+        result = main.TRUE
         if (check1 or check2 or check3 or check4):
-            result = main.TRUE
-        utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail="Exceptions found")
+            result = main.FALSE
+            count = len(check1.splitlines()) + len(check2.splitlines()) + len(check3.splitlines()) + len(check4.splitlines())
+        utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail=str(count) + " Exceptions were found in the logs")