CHO will now not always select ONOS1 to be master of all switches
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 4861e05..a04309e 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -615,4 +615,13 @@
elif i == 3:
main.log.error("TIMEOUT!")
return main.FALSE
-
+
+ def tcpdump(self):
+ self.handle.sendline("")
+ self.handle.expect("\$")
+ self.handle.sendline("sudo tcpdump -n -i eth0 -s0 -w onos-logs/tcpdump &")
+
+ def kill_tcpdump(self):
+ self.handle.sendline("")
+ self.handle.expect("\$")
+ self.handle.sendline("sudo kill -9 `ps -ef | grep \"tcpdump -n\" | grep -v grep | awk '{print $2}'`")
diff --git a/TestON/tests/OnosCHO4nodes/OnosCHO4nodes.py b/TestON/tests/OnosCHO4nodes/OnosCHO4nodes.py
index 763791d..6b4221a 100644
--- a/TestON/tests/OnosCHO4nodes/OnosCHO4nodes.py
+++ b/TestON/tests/OnosCHO4nodes/OnosCHO4nodes.py
@@ -9,12 +9,34 @@
#Tests the startup of Zookeeper1, Cassandra1, and ONOS1 to be certain that all started up successfully
def CASE1(self,main) : #Check to be sure ZK, Cass, and ONOS are up, then get ONOS version
import time
- main.log.report("Setting up fresh database")
main.ONOS1.stop()
main.ONOS2.stop()
main.ONOS3.stop()
main.ONOS4.stop()
- data = main.Cassandra1.isup()
+ main.Cassandra1.start()
+ main.Cassandra2.start()
+ main.Cassandra3.start()
+ main.Cassandra4.start()
+ time.sleep(20)
+ main.ONOS1.drop_keyspace()
+ main.ONOS1.start()
+ time.sleep(10)
+ main.ONOS2.start()
+ main.ONOS3.start()
+ main.ONOS4.start()
+ main.ONOS1.start_rest()
+ time.sleep(5)
+ test= main.ONOS1.rest_status()
+ if test == main.FALSE:
+ main.ONOS1.start_rest()
+ main.ONOS1.get_version()
+ main.log.report("Startup check Zookeeper1, Cassandra1, and ONOS1 connections")
+ main.case("Checking if the startup was clean...")
+ main.step("Testing startup Zookeeper")
+ data = main.Zookeeper1.isup()
+ utilities.assert_equals(expect=main.TRUE,actual=data,onpass="Zookeeper is up!",onfail="Zookeeper is down...")
+ main.step("Testing startup Cassandra")
+ data = main.Cassandra1.isup()
if data == main.FALSE:
main.Cassandra1.stop()
main.Cassandra2.stop()
@@ -22,46 +44,34 @@
main.Cassandra4.stop()
time.sleep(5)
-
+
main.Cassandra1.start()
main.Cassandra2.start()
main.Cassandra3.start()
main.Cassandra4.start()
utilities.assert_equals(expect=main.TRUE,actual=data,onpass="Cassandra is up!",onfail="Cassandra is down...")
- main.ONOS1.drop_keyspace()
- main.ONOS1.start()
- main.ONOS2.start()
- main.ONOS3.start()
- main.ONOS4.start()
- '''
- For embedded ONOS
- main.Cassandra1.stop()
- main.Cassandra1.stop()
- main.Cassandra1.stop()
- main.Cassandra1.stop()
- main.log.report("Starting embedded ONOS")
- main.ONOS1.start_embedded()
- time.sleep(30)
- main.ONOS2.start_embedded()
- main.ONOS3.start_embedded()
- main.ONOS4.start_embedded()
- '''
- main.ONOS1.start_rest()
- time.sleep(30)
- test= main.ONOS1.rest_status()
- if test == main.FALSE:
- main.ONOS1.start_rest()
- main.ONOS1.get_version()
- main.log.report("Startup check Zookeeper1, Cassandra1, and ONOS1 connections")
- main.case("Checking if the startup was clean...")
- main.step("Testing startup Zookeeper")
- main.step("Testing startup Cassandra")
- data = main.Cassandra1.isup()
- utilities.assert_equals(expect=main.TRUE,actual=data,onpass="Cassandra is up!",onfail="Cassandra is down...")
- data = main.Zookeeper1.isup()
- utilities.assert_equals(expect=main.TRUE,actual=data,onpass="Zookeeper is up!",onfail="Zookeeper is down...")
main.step("Testing startup ONOS")
- data = main.ONOS1.isup()
+ data = main.ONOS1.isup()
+ data = data and main.ONOS2.isup()
+ data = data and main.ONOS3.isup()
+ data = data and main.ONOS4.isup()
+ if data == main.FALSE:
+ main.log.report("Something is funny... restarting ONOS")
+ main.ONOS1.stop()
+ main.ONOS2.stop()
+ main.ONOS3.stop()
+ main.ONOS4.stop()
+ time.sleep(5)
+ main.ONOS1.start()
+ time.sleep(10)
+ main.ONOS2.start()
+ main.ONOS3.start()
+ main.ONOS4.start()
+ data = main.ONOS1.isup()
+ #main.ONOS1.tcpdump()
+ #main.ONOS2.tcpdump()
+ #main.ONOS3.tcpdump()
+ #main.ONOS4.tcpdump()
utilities.assert_equals(expect=main.TRUE,actual=data,onpass="ONOS is up and running!",onfail="ONOS didn't start...")
#**********************************************************************************************************************************************************************************************
@@ -102,6 +112,13 @@
time.sleep(1)
main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'])
main.Mininet1.get_sw_controller("s1")
+
+ for i in range(9):
+ if result == main.FALSE:
+ time.sleep(3)
+ result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
+ else:
+ break
# **********************************************************************************************************************************************************************************************
#Add Flows
@@ -111,6 +128,15 @@
def CASE3(self,main) : #Delete any remnant flows, then add flows, and time how long it takes flow tables to update
main.log.report("Delete any flows from previous tests, then add flows from FLOWDEF file, then wait for switch flow tables to update")
import time
+
+ result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
+ for i in range(9):
+ if result == main.FALSE:
+ time.sleep(3)
+ result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
+ else:
+ break
+
main.case("Taking care of these flows!")
main.step("Cleaning out any leftover flows...")
main.ONOS1.delete_flow("all")
@@ -137,11 +163,10 @@
break
endTime = time.time()
if result1 == main.TRUE:
- main.log.report("\n\t\t\t\tTime to add flows: "+str(round(endTime-strtTime,2)))
+ main.log.report("\n\t\t\t\tTime to add flows: "+str(round(endTime-strtTime,2))+" seconds")
else:
main.log.report("\tFlows failed check")
- strtTime = time.time()
count = 1
i = 6
while i < 16 :
@@ -150,7 +175,7 @@
if ping == main.FALSE and count < 3:
count = count + 1
i = 6
- main.log.info("Ping failed, making attempt number "+str(count)+" in 10 seconds")
+ main.log.report("Ping failed, making attempt number "+str(count)+" in 10 seconds")
time.sleep(10)
elif ping == main.FALSE and count ==3:
main.log.error("Ping test failed")
@@ -159,15 +184,13 @@
elif ping == main.TRUE:
i = i + 1
result2 = main.TRUE
- endTime = time.time()
if result2 == main.TRUE:
- main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2)))
+ main.log.info("Flows successfully added")
else:
main.log.report("\tPING TEST FAIL")
- result3 = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
main.step("Verifying the result")
- utilities.assert_equals(expect=main.TRUE,actual=result1 and result2 and result3,onpass="Flow check PASS",onfail="Flow check FAIL")
+ utilities.assert_equals(expect=main.TRUE,actual=result1 and result2,onpass="Flow check PASS",onfail="Flow check FAIL")
#**********************************************************************************************************************************************************************************************
#This test case removes Controllers 2,3, and 4 then performs a ping test.
@@ -176,31 +199,27 @@
#If the ping test fails 6 times, then the test case will return false
def CASE4(self,main) :
- main.log.report("Remove all but one ONOS node then ping until all hosts are reachable or fail after 6 attempts")
+ main.log.report("Remove all but one ONOS then ping until all hosts are reachable or fail after 6 attempts")
import time
import random
random.seed(None)
- rand = random.randrange(4)
-
- if rand == 0:
- main.log.info("Controller 1 will be the lone coontroller")
+ num = random.randint(1,4)
+ if num == 1:
ip = main.params['CTRL']['ip1']
port = main.params['CTRL']['port1']
- elif rand == 1:
- main.log.info("Controller 2 will be the lone controller")
+ elif num == 2:
ip = main.params['CTRL']['ip2']
port = main.params['CTRL']['port2']
- elif rand == 2:
- main.log.info("Controller 3 will be the lone controller")
+ elif num == 3:
ip = main.params['CTRL']['ip3']
port = main.params['CTRL']['port3']
- elif rand == 3:
- main.log.info("Controller 4 will be the lone controller")
+ else:
ip = main.params['CTRL']['ip4']
port = main.params['CTRL']['port4']
+ main.log.report("ONOS"+str(num)+" will be the sole controller")
for i in range(25):
if i < 15:
j=i+1
@@ -208,14 +227,12 @@
else:
j=i+16
main.Mininet1.assign_sw_controller(sw=str(j),ip1=ip,port1=port)
-
+
+ strtTime = time.time()
result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
-
- strtTime = time.time()
-
- for i in range(2):
+ for i in range(9):
if result == main.FALSE:
- time.sleep(5)
+ time.sleep(3)
result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
else:
break
@@ -228,7 +245,7 @@
if ping == main.FALSE and count < 6:
count = count + 1
i = 6
- main.log.info("Ping failed, making attempt number "+str(count)+" in 5 seconds")
+ main.log.report("Ping failed, making attempt number "+str(count)+" in 5 seconds")
time.sleep(5)
elif ping == main.FALSE and count ==6:
main.log.error("Ping test failed")
@@ -239,7 +256,7 @@
result = main.TRUE
endTime = time.time()
if result == main.TRUE:
- main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2)))
+ main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds")
else:
main.log.report("\tPING TEST FAIL")
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE")
@@ -257,14 +274,12 @@
else:
j=i+16
main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4'])
-
+
+ strtTime = time.time()
result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
-
- strtTime = time.time()
-
- for i in range(2):
+ for i in range(9):
if result == main.FALSE:
- time.sleep(5)
+ time.sleep(3)
result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
else:
break
@@ -277,7 +292,7 @@
if ping == main.FALSE and count < 6:
count = count + 1
i = 6
- main.log.info("Ping failed, making attempt number "+str(count)+" in 5 seconds")
+ main.log.report("Ping failed, making attempt number "+str(count)+" in 5 seconds")
time.sleep(5)
elif ping == main.FALSE and count ==6:
main.log.error("Ping test failed")
@@ -288,7 +303,7 @@
result = main.TRUE
endTime = time.time()
if result == main.TRUE:
- main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2)))
+ main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds")
else:
main.log.report("\tPING TEST FAILED")
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE")
@@ -301,15 +316,13 @@
import time
main.case("Bringing Link down... ")
result = main.Mininet1.link(END1=main.params['LINK']['begin'],END2=main.params['LINK']['end'],OPTION="down")
- time.sleep(15)
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link DOWN!",onfail="Link not brought down...")
- strtTime = time.time()
-
+ strtTime = time.time()
result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],str(int(main.params['NR_Links'])-2))
- for i in range(2):
+ for i in range(9):
if result == main.FALSE:
- time.sleep(5)
+ time.sleep(3)
result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],str(int(main.params['NR_Links'])-2))
else:
break
@@ -321,7 +334,7 @@
ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25))
if ping == main.FALSE and count < 10:
count = count + 1
- main.log.info("Ping failed, making attempt number "+str(count)+" in 5 seconds")
+ main.log.report("Ping failed, making attempt number "+str(count)+" in 5 seconds")
i = 6
time.sleep(5)
elif ping == main.FALSE and count == 10:
@@ -333,7 +346,7 @@
result = main.TRUE
endTime = time.time()
if result == main.TRUE:
- main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2)))
+ main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds")
else:
main.log.report("\tPING TEST FAILED")
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE")
@@ -346,15 +359,13 @@
import time
main.case("Bringing Link up... ")
result = main.Mininet1.link(END1=main.params['LINK']['begin'],END2=main.params['LINK']['end'],OPTION="up")
- time.sleep(15)
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link UP!",onfail="Link not brought up...")
-
- strtTime = time.time()
-
+
+ strtTime = time.time()
result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
- for i in range(2):
+ for i in range(9):
if result == main.FALSE:
- time.sleep(5)
+ time.sleep(3)
result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links'])
else:
break
@@ -367,10 +378,10 @@
ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25))
if ping == main.FALSE and count < 10:
count = count + 1
- main.log.info("Ping failed, making attempt number "+str(count)+" in 5 seconds")
+ main.log.report("Ping failed, making attempt number "+str(count)+" in 5 seconds")
i = 6
time.sleep(5)
- elif ping == main.FALSE and count == 10:
+ elif ping == main.FALSE and count ==10:
main.log.error("Ping test failed")
i = 17
result = main.FALSE
@@ -379,7 +390,7 @@
result = main.TRUE
endTime = time.time()
if result == main.TRUE:
- main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2)))
+ main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds")
else:
main.log.report("\tPING TESTS FAILED")
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE")
@@ -407,11 +418,18 @@
Reststatus, Hoststatus = restcall.find_host(RestIP1,RestPort,url,mac)
try:
attachedSW = Hoststatus[0]['attachmentPoint'][0]['switchDPID']
- except IndexError:
+ ip_found = Hoststatus[0]['ipv4'][0]
+ except:
Reststatus = 0
+
if Reststatus == 1:
main.log.report("\tFound host " + host + " attached to switchDPID = " + attachedSW)
- result = main.TRUE
+ if ip_found != None:
+ main.log.report("\t IP discovered is ip_found ( " + ip_found + " ).")
+ result = main.TRUE
+ else:
+ main.log.report("\t Found host attached to switch, but no IP address discovered.")
+ result = main.FALSE
else:
main.log.report("\t Host " + host + " with MAC:" + str(mac) + " does not exist. FAILED")
result = main.FALSE
@@ -427,7 +445,7 @@
Reststatus, Hoststatus = restcall.find_host(RestIP1,RestPort,url,mac)
try:
attachedSW = Hoststatus[0]['attachmentPoint'][0]['switchDPID']
- except IndexError:
+ except:
Reststatus = 0
if Reststatus == 0:
main.log.report("Attempt to yank out s1-eth1 from s1 sucessfully")
@@ -446,14 +464,20 @@
Reststatus, Hoststatus = restcall.find_host(RestIP2,RestPort,url,mac)
try:
attachedSW = Hoststatus[0]['attachmentPoint'][0]['switchDPID']
- except IndexError:
+ ip_found = Hoststatus[0]['ipv4'][0]
+ except:
Reststatus = 0
if Reststatus == 0:
main.log.report("Attempt to plug s1-eth1 to s6 FAILED")
result = main.FALSE
elif attachedSW == "00:00:00:00:00:00:00:06":
main.log.report("Attempt to plug s1-eht1 to s6 succeded.")
- result = main.TRUE
+ if ip_found != None:
+ main.log.report("\t IP discovered is ip_found ( " + ip_found + " ).")
+ result = main.TRUE
+ else:
+ main.log.report("\t Found host attached to switch, but no IP address discovered.")
+ result = main.FALSE
else:
main.log.report( "FAILED to attach s1-eth1 to s6 correctly!")
result = main.FALSE
@@ -469,14 +493,20 @@
Reststatus, Hoststatus = restcall.find_host(RestIP1,RestPort,url,mac)
try:
attachedSW = Hoststatus[0]['attachmentPoint'][0]['switchDPID']
- except IndexError:
+ ip_found = Hoststatus[0]['ipv4'][0]
+ except:
Reststatus = 0
if Reststatus == 0:
main.log.report("Attempt to plug s1-eth1 back to s1 FAILED")
result = main.FALSE
elif attachedSW == "00:00:00:00:00:00:00:01":
main.log.report("Attempt to plug s1-eht1 back to s1 succeded.")
- result = main.TRUE
+ if ip_found != None:
+ main.log.report("\t IP discovered is ip_found ( " + ip_found + " ).")
+ result = main.TRUE
+ else:
+ main.log.report("\t Found host attached to switch, but no IP address discovered.")
+ result = main.FALSE
else:
main.log.report( "FAIL to attach s1-eth1 to s1 correctly!")
result = main.FALSE
@@ -484,3 +514,5 @@
utilities.assert_equals(expect=main.TRUE,actual=result,onpass="DEVICE DISCOVERY TEST PASSED PLUG/UNPLUG/MOVE TEST",onfail="DEVICE DISCOVERY TEST FAILED")
+
+