Address comments from code review
- Removed all grep_str's from onosclidriver
- fix some sendline issues
- refactor HA tests a bit
- fix typos
diff --git a/TestON/drivers/common/cli/emulator/lincoedriver.py b/TestON/drivers/common/cli/emulator/lincoedriver.py
index 4d2cc01..6f7ce3a 100644
--- a/TestON/drivers/common/cli/emulator/lincoedriver.py
+++ b/TestON/drivers/common/cli/emulator/lincoedriver.py
@@ -293,8 +293,8 @@
'''
try:
#Send CTRL+C twice to exit CLI
- self.handle.sendline("\x03")
- self.handle.sendline("\x03")
+ self.handle.send("\x03")
+ self.handle.send("\x03")
self.handle.expect("\$")
except pexpect.EOF:
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index e2570ac..2f31b7d 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -672,7 +672,7 @@
returns: main.FASLE on an error, else main.TRUE
'''
dpid = kwargs.get('dpid', '')
- command = "addswitch " + sw + " " + str(dpid)
+ command = "addswitch " + str( sw ) + " " + str( dpid )
try:
response = self.execute(cmd=command,prompt="mininet>",timeout=10)
if re.search("already exists!", response):
@@ -700,7 +700,7 @@
switchname = name of the switch as a string
returns: main.FASLE on an error, else main.TRUE
'''
- command = "delswitch " + sw
+ command = "delswitch " + str( sw )
try:
response = self.execute(cmd=command,prompt="mininet>",timeout=10)
if re.search("no switch named", response):
@@ -730,7 +730,7 @@
node2 = the string node name of the second endpoint of the link
returns: main.FASLE on an error, else main.TRUE
'''
- command = "addlink " + node1 + " " + node2
+ command = "addlink " + str( node1 ) + " " + str( node2 )
try:
response = self.execute(cmd=command,prompt="mininet>",timeout=10)
if re.search("doesnt exist!", response):
@@ -759,7 +759,7 @@
node2 = the string node name of the second endpoint of the link
returns: main.FASLE on an error, else main.TRUE
'''
- command = "dellink " + node1 + " " + node2
+ command = "dellink " + str( node1 ) + " " + str( node2 )
try:
response = self.execute(cmd=command,prompt="mininet>",timeout=10)
if re.search("no node named", response):
@@ -791,7 +791,7 @@
returns: main.FASLE on an error, else main.TRUE
'''
switch = kwargs.get('switch', '')
- command = "addhost " + hostname + " " + switch
+ command = "addhost " + str( hostname ) + " " + str( switch )
try:
response = self.execute(cmd=command,prompt="mininet>",timeout=10)
if re.search("already exists!", response):
@@ -822,7 +822,7 @@
hostname = the string hostname
returns: main.FASLE on an error, else main.TRUE
'''
- command = "delhost " + hostname
+ command = "delhost " + str( hostname )
try:
response = self.execute(cmd=command,prompt="mininet>",timeout=10)
if re.search("no host named", response):
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index dd54d45..4916ba5 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -90,7 +90,7 @@
self.handle.expect("Confirm")
self.handle.sendline("yes")
self.handle.expect("\$")
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline("exit")
self.handle.expect("closed")
@@ -194,13 +194,7 @@
else:
#If failed, send ctrl+c to process and try again
main.log.info("Starting CLI failed. Retrying...")
- self.handle.sendline("\x03")
- i = self.handle.expect(["onos>",pexpect.TIMEOUT],
- timeout=30)
- #Send ctrl+d to exit the onos> prompt that was
- #not successful
- self.handle.sendline("\x04")
- self.handle.expect("\$")
+ self.handle.send("\x03")
self.handle.sendline("onos -w "+str(ONOS_ip))
i = self.handle.expect(["onos>",pexpect.TIMEOUT],
timeout=30)
@@ -467,27 +461,21 @@
main.log.info(self.name+" ::::::")
main.cleanup()
main.exit()
-
- def devices(self, json_format=True, grep_str=""):
+
+ def devices(self, json_format=True):
'''
Lists all infrastructure devices or switches
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
if json_format:
- if not grep_str:
- self.handle.sendline("devices -j")
- self.handle.expect("devices -j")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("devices -j | grep '"+
- str(grep_str)+"'")
- self.handle.expect("devices -j | grep '"+str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("devices -j")
+ self.handle.expect("devices -j")
+ self.handle.expect("onos>")
handle = self.handle.before
'''
handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
@@ -503,13 +491,8 @@
#print "repr(handle1) = ", repr(handle1)
return handle1
else:
- if not grep_str:
- self.handle.sendline("devices")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("devices | grep '"+
- str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("devices")
+ self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
return handle
@@ -525,34 +508,28 @@
main.cleanup()
main.exit()
- def links(self, json_format=True, grep_str=""):
+ def links(self, json_format=True):
'''
Lists all core links
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
if json_format:
- if not grep_str:
- self.handle.sendline("links -j")
- self.handle.expect("links -j")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("links -j | grep '"+
- str(grep_str)+"'")
- self.handle.expect("links -j | grep '"+str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("links -j")
+ self.handle.expect("links -j")
+ self.handle.expect("onos>")
handle = self.handle.before
'''
handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
- So we take off that escape sequence using
+ So we take off that escape sequence using
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
- handle1 = ansi_escape.sub('', handle)
+ handle1 = ansi_escape.sub('', handle)
'''
#print "repr(handle) =", repr(handle)
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -560,13 +537,8 @@
#print "repr(handle1) = ", repr(handle1)
return handle1
else:
- if not grep_str:
- self.handle.sendline("links")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("links | grep '"+
- str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("links")
+ self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
return handle
@@ -583,34 +555,28 @@
main.exit()
- def ports(self, json_format=True, grep_str=""):
+ def ports(self, json_format=True):
'''
Lists all ports
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
if json_format:
- if not grep_str:
- self.handle.sendline("ports -j")
- self.handle.expect("ports -j")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("ports -j | grep '"+
- str(grep_str)+"'")
- self.handle.expect("ports -j | grep '"+str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("ports -j")
+ self.handle.expect("ports -j")
+ self.handle.expect("onos>")
handle = self.handle.before
'''
handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
- So we take off that escape sequence using the following commads:
+ So we take off that escape sequence using the following commads:
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
- handle1 = ansi_escape.sub('', handle)
+ handle1 = ansi_escape.sub('', handle)
'''
#print "repr(handle) =", repr(handle)
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -619,20 +585,13 @@
return handle1
else:
- if not grep_str:
- self.handle.sendline("ports")
- self.handle.expect("onos>")
- self.handle.sendline("")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("ports | grep '"+
- str(grep_str)+"'")
- self.handle.expect("onos>")
- self.handle.sendline("")
- self.handle.expect("onos>")
+ self.handle.sendline("ports")
+ self.handle.expect("onos>")
+ self.handle.sendline("")
+ self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
- return handle
+ return handle
except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
@@ -646,11 +605,11 @@
main.exit()
- def roles(self, json_format=True, grep_str=""):
+ def roles(self, json_format=True):
'''
Lists all devices and the controllers with roles assigned to them
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
@@ -686,7 +645,7 @@
self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
- return handle
+ return handle
except pexpect.EOF:
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
@@ -774,35 +733,29 @@
main.log.info(self.name+" ::::::")
main.cleanup()
main.exit()
-
- def hosts(self, json_format=True, grep_str=""):
+
+ def hosts(self, json_format=True):
'''
- Lists all discovered hosts
+ Lists all discovered hosts
Optional argument:
- * grep_str - pass in a string to grep
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
self.handle.expect("onos>")
-
+
if json_format:
- if not grep_str:
- self.handle.sendline("hosts -j")
- self.handle.expect("hosts -j")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("hosts -j | grep '"+
- str(grep_str)+"'")
- self.handle.expect("hosts -j | grep '"+str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("hosts -j")
+ self.handle.expect("hosts -j")
+ self.handle.expect("onos>")
handle = self.handle.before
'''
handle variable here contains some ANSI escape color code sequences at the end which are invisible in the print command output
To make that escape sequence visible, use repr() function. The repr(handle) output when printed shows the ANSI escape sequences.
In json.loads(somestring), this somestring variable is actually repr(somestring) and json.loads would fail with the escape sequence.
- So we take off that escape sequence using
+ So we take off that escape sequence using
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
- handle1 = ansi_escape.sub('', handle)
+ handle1 = ansi_escape.sub('', handle)
'''
#print "repr(handle) =", repr(handle)
ansi_escape = re.compile(r'\r\r\n\x1b[^m]*m')
@@ -810,13 +763,8 @@
#print "repr(handle1) = ", repr(handle1)
return handle1
else:
- if not grep_str:
- self.handle.sendline("hosts")
- self.handle.expect("onos>")
- else:
- self.handle.sendline("hosts | grep '"+
- str(grep_str)+"'")
- self.handle.expect("onos>")
+ self.handle.sendline("hosts")
+ self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
return handle
@@ -925,7 +873,7 @@
self.handle.expect("onos>")
handle = self.handle.before
- print "handle =", handle
+ #print "handle =", handle
main.log.info("Intent installed between "+
str(host_id_one) + " and " + str(host_id_two))
@@ -1168,7 +1116,7 @@
main.cleanup()
main.exit()
- def intents(self, json_format = False):
+ def intents(self, json_format = True):
'''
Optional:
* json_format: enable output formatting in json
@@ -1206,7 +1154,7 @@
main.cleanup()
main.exit()
- def flows(self, json_format = False):
+ def flows(self, json_format = True):
'''
Optional:
* json_format: enable output formatting in json
@@ -1649,6 +1597,8 @@
def clusters(self, json_format=True):
'''
Lists all clusters
+ Optional argument:
+ * json_format - boolean indicating if you want output in json
'''
try:
self.handle.sendline("")
@@ -1676,7 +1626,7 @@
#print "repr(handle1) = ", repr(handle1)
return handle1
else:
- self.handle.sendline("links")
+ self.handle.sendline("clusters")
self.handle.expect("onos>")
handle = self.handle.before
#print "handle =",handle
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index f733d35..1f52be1 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -82,7 +82,7 @@
'''
response = ''
try:
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline("exit")
self.handle.expect("closed")
@@ -164,7 +164,7 @@
self.handle.sendline("cd "+ self.home)
self.handle.expect("\$")
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline("mvn clean install")
self.handle.expect("mvn clean install")
@@ -195,7 +195,7 @@
for line in self.handle.before.splitlines():
if "Total time:" in line:
main.log.info(line)
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$", timeout=60)
return main.TRUE
elif i == 4:
@@ -392,11 +392,11 @@
self.handle.sendline("export TERM=xterm-256color")
self.handle.expect("xterm-256color")
self.handle.expect("\$")
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline("cd " + self.home + "; git log -1 --pretty=fuller --decorate=short | grep -A 6 \"commit\" --color=never")
#NOTE: for some reason there are backspaces inserted in this phrase when run from Jenkins on some tests
- #self.handle.expect("--color=never")
+ self.handle.expect("never")
self.handle.expect("\$")
response=(self.name +": \n"+ str(self.handle.before + self.handle.after))
self.handle.sendline("cd " + self.home)
@@ -790,7 +790,6 @@
self.handle.sendline("")
self.handle.expect("\$")
self.handle.sendline( "onos-uninstall "+str(node_ip) )
- self.handle.expect( "onos-uninstall "+str(node_ip) )
self.handle.expect("\$")
main.log.info("ONOS "+node_ip+" was uninstalled")
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index e54c663..7bf7667 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -76,7 +76,7 @@
elif i==6:
main.log.info("Password not required logged in")
- self.handle.sendline("\n")
+ self.handle.sendline("")
self.handle.expect('>|#|\$')
return self.handle
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 49da6e9..47dc744 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -245,7 +245,10 @@
mastership_check = main.TRUE
for i in range (1,29):
response = main.Mininet1.get_sw_controller("s"+str(i))
- main.log.info(repr(response))
+ try:
+ main.log.info(str(response))
+ except:
+ main.log.info(repr(response))
if re.search("tcp:"+ONOS1_ip,response)\
and re.search("tcp:"+ONOS2_ip,response)\
and re.search("tcp:"+ONOS3_ip,response)\
@@ -664,15 +667,19 @@
main.ONOSbench.onos_kill(ONOS7_ip)
main.step("Checking if ONOS is up yet")
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
- onos2_isup = main.ONOSbench.isup(ONOS2_ip)
- onos3_isup = main.ONOSbench.isup(ONOS3_ip)
- onos4_isup = main.ONOSbench.isup(ONOS4_ip)
- onos5_isup = main.ONOSbench.isup(ONOS5_ip)
- onos6_isup = main.ONOSbench.isup(ONOS6_ip)
- onos7_isup = main.ONOSbench.isup(ONOS7_ip)
- onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
- and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ count = 0
+ onos_isup_result = main.FALSE
+ while onos_isup_result == main.FALSE and count < 10:
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ count = count + 1
# TODO: if it becomes an issue, we can retry this step a few times
@@ -924,43 +931,6 @@
break
MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
-
-
main.step("Comparing ONOS topology to MN")
devices_results = main.TRUE
ports_results = main.TRUE
@@ -968,8 +938,46 @@
topo_result = main.FALSE
start_time = time.time()
elapsed = 0
+ count = 0
while topo_result == main.FALSE and elapsed < 120:
+ count = count + 1
try:
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+
for controller in range(7): #TODO parameterize the number of controllers
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
@@ -1003,11 +1011,13 @@
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- elapsed = time.time()-start_time()
+ topo_result = devices_results and ports_results and links_results
+ elapsed = time.time() - start_time
time_threshold = elapsed < 1
- topo_result = devices_results and ports_results and links_results and time_threshold
+ topo_result = topo_result and time_threshold
#TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " + str(elapsed) + " seconds")
+ main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index d68dc0b..2af775d 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -245,7 +245,10 @@
mastership_check = main.TRUE
for i in range (1,29):
response = main.Mininet1.get_sw_controller("s"+str(i))
- main.log.info(repr(response))
+ try:
+ main.log.info(str(response))
+ except:
+ main.log.info(repr(response))
if re.search("tcp:"+ONOS1_ip,response)\
and re.search("tcp:"+ONOS2_ip,response)\
and re.search("tcp:"+ONOS3_ip,response)\
@@ -661,10 +664,14 @@
main.ONOSbench.onos_kill(ONOS3_ip)
main.step("Checking if ONOS is up yet")
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
- onos2_isup = main.ONOSbench.isup(ONOS2_ip)
- onos3_isup = main.ONOSbench.isup(ONOS3_ip)
- onos_isup_result = onos1_isup and onos2_isup and onos3_isup
+ count = 0
+ onos_isup_result = main.FALSE
+ while onos_isup_result == main.FALSE and count < 10:
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup
+ count = count + 1
# TODO: if it becomes an issue, we can retry this step a few times
@@ -924,43 +931,6 @@
break
MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
-
-
main.step("Comparing ONOS topology to MN")
devices_results = main.TRUE
ports_results = main.TRUE
@@ -968,8 +938,46 @@
topo_result = main.FALSE
start_time = time.time()
elapsed = 0
+ count = 0
while topo_result == main.FALSE and elapsed < 120:
+ count = count + 1
try:
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+
for controller in range(7): #TODO parameterize the number of controllers
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
@@ -1003,11 +1011,13 @@
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- elapsed = time.time()-start_time()
+ topo_result = devices_results and ports_results and links_results
+ elapsed = time.time() - start_time
time_threshold = elapsed < 1
- topo_result = devices_results and ports_results and links_results and time_threshold
+ topo_result = topo_result and time_threshold
#TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " + str(elapsed) + " seconds")
+ main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 54e9d3f..614d514 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -40,6 +40,7 @@
import time
main.log.report("ONOS HA Sanity test - initialization")
main.case("Setting up test environment")
+ #TODO: save all the timers and output them for plotting
# load some vairables from the params file
PULL_CODE = False
@@ -83,7 +84,7 @@
cell_result = main.ONOSbench.set_cell(cell_name)
verify_result = main.ONOSbench.verify_cell()
- #FIXME:this is short term fix
+ #FIXME:this is short term fix
main.log.report("Removing raft logs")
main.ONOSbench.onos_remove_raft_logs()
main.log.report("Uninstalling ONOS")
@@ -146,29 +147,32 @@
main.step("Checking if ONOS is up yet")
#TODO: Refactor
# check bundle:list?
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
- if not onos1_isup:
- main.log.report("ONOS1 didn't start!")
- onos2_isup = main.ONOSbench.isup(ONOS2_ip)
- if not onos2_isup:
- main.log.report("ONOS2 didn't start!")
- onos3_isup = main.ONOSbench.isup(ONOS3_ip)
- if not onos3_isup:
- main.log.report("ONOS3 didn't start!")
- onos4_isup = main.ONOSbench.isup(ONOS4_ip)
- if not onos4_isup:
- main.log.report("ONOS4 didn't start!")
- onos5_isup = main.ONOSbench.isup(ONOS5_ip)
- if not onos5_isup:
- main.log.report("ONOS5 didn't start!")
- onos6_isup = main.ONOSbench.isup(ONOS6_ip)
- if not onos6_isup:
- main.log.report("ONOS6 didn't start!")
- onos7_isup = main.ONOSbench.isup(ONOS7_ip)
- if not onos7_isup:
- main.log.report("ONOS7 didn't start!")
- onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
- and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ for i in range(2):
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if not onos1_isup:
+ main.log.report("ONOS1 didn't start!")
+ onos2_isup = main.ONOSbench.isup(ONOS2_ip)
+ if not onos2_isup:
+ main.log.report("ONOS2 didn't start!")
+ onos3_isup = main.ONOSbench.isup(ONOS3_ip)
+ if not onos3_isup:
+ main.log.report("ONOS3 didn't start!")
+ onos4_isup = main.ONOSbench.isup(ONOS4_ip)
+ if not onos4_isup:
+ main.log.report("ONOS4 didn't start!")
+ onos5_isup = main.ONOSbench.isup(ONOS5_ip)
+ if not onos5_isup:
+ main.log.report("ONOS5 didn't start!")
+ onos6_isup = main.ONOSbench.isup(ONOS6_ip)
+ if not onos6_isup:
+ main.log.report("ONOS6 didn't start!")
+ onos7_isup = main.ONOSbench.isup(ONOS7_ip)
+ if not onos7_isup:
+ main.log.report("ONOS7 didn't start!")
+ onos_isup_result = onos1_isup and onos2_isup and onos3_isup\
+ and onos4_isup and onos5_isup and onos6_isup and onos7_isup
+ if onos_isup_result == main.TRUE:
+ break
# TODO: if it becomes an issue, we can retry this step a few times
@@ -198,9 +202,9 @@
onfail="Test startup NOT successful")
- #if case1_result==main.FALSE:
- # main.cleanup()
- # main.exit()
+ if case1_result==main.FALSE:
+ main.cleanup()
+ main.exit()
def CASE2(self,main) :
'''
@@ -246,7 +250,10 @@
mastership_check = main.TRUE
for i in range (1,29):
response = main.Mininet1.get_sw_controller("s"+str(i))
- main.log.info(repr(response))
+ try:
+ main.log.info(str(response))
+ except:
+ main.log.info(repr(response))
if re.search("tcp:"+ONOS1_ip,response)\
and re.search("tcp:"+ONOS2_ip,response)\
and re.search("tcp:"+ONOS3_ip,response)\
@@ -891,43 +898,6 @@
break
MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- hosts.append( main.ONOScli2.hosts() )
- hosts.append( main.ONOScli3.hosts() )
- hosts.append( main.ONOScli4.hosts() )
- hosts.append( main.ONOScli5.hosts() )
- hosts.append( main.ONOScli6.hosts() )
- hosts.append( main.ONOScli7.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
- links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
-
-
main.step("Comparing ONOS topology to MN")
devices_results = main.TRUE
ports_results = main.TRUE
@@ -935,8 +905,48 @@
topo_result = main.FALSE
start_time = time.time()
elapsed = 0
+ count = 0
while topo_result == main.FALSE and elapsed < 120:
+ print "cond 1:" + str(topo_result == main.FALSE)
+ print "cond 2:" + str(elapsed < 120)
+ count = count + 1
try:
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ devices.append( main.ONOScli2.devices() )
+ devices.append( main.ONOScli3.devices() )
+ devices.append( main.ONOScli4.devices() )
+ devices.append( main.ONOScli5.devices() )
+ devices.append( main.ONOScli6.devices() )
+ devices.append( main.ONOScli7.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ hosts.append( main.ONOScli2.hosts() )
+ hosts.append( main.ONOScli3.hosts() )
+ hosts.append( main.ONOScli4.hosts() )
+ hosts.append( main.ONOScli5.hosts() )
+ hosts.append( main.ONOScli6.hosts() )
+ hosts.append( main.ONOScli7.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ ports.append( main.ONOScli2.ports() )
+ ports.append( main.ONOScli3.ports() )
+ ports.append( main.ONOScli4.ports() )
+ ports.append( main.ONOScli5.ports() )
+ ports.append( main.ONOScli6.ports() )
+ ports.append( main.ONOScli7.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
+ links.append( main.ONOScli2.links() )
+ links.append( main.ONOScli3.links() )
+ links.append( main.ONOScli4.links() )
+ links.append( main.ONOScli5.links() )
+ links.append( main.ONOScli6.links() )
+ links.append( main.ONOScli7.links() )
+
for controller in range(7): #TODO parameterize the number of controllers
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
@@ -970,11 +980,13 @@
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- elapsed = time.time()-start_time()
+ topo_result = devices_results and ports_results and links_results
+ elapsed = time.time() - start_time
time_threshold = elapsed < 1
- topo_result = devices_results and ports_results and links_results and time_threshold
+ topo_result = topo_result and time_threshold
#TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " + str(elapsed) + " seconds")
+ main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")
diff --git a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
index f39eb12..e6f2fde 100644
--- a/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
+++ b/TestON/tests/SingleInstanceHATestRestart/SingleInstanceHATestRestart.py
@@ -172,7 +172,10 @@
mastership_check = main.TRUE
for i in range (1,29):
response = main.Mininet1.get_sw_controller("s"+str(i))
- main.log.info(repr(response))
+ try:
+ main.log.info(str(response))
+ except:
+ main.log.info(repr(response))
if re.search("tcp:"+ONOS1_ip,response):
mastership_check = mastership_check and main.TRUE
else:
@@ -422,24 +425,32 @@
def CASE6(self,main) :
'''
- The Failure case. Since this is the Sanity test, we do nothing.
+ The Failure case.
'''
+ import time
main.log.report("Restart ONOS node")
main.log.case("Restart ONOS node")
main.ONOSbench.onos_kill(ONOS1_ip)
+ start = time.time()
main.step("Checking if ONOS is up yet")
- onos1_isup = main.ONOSbench.isup(ONOS1_ip)
- # TODO: if it becomes an issue, we can retry this step a few times
+ count = 0
+ while count < 10
+ onos1_isup = main.ONOSbench.isup(ONOS1_ip)
+ if onos1_isup == main.TRUE:
+ elapsed = time.time() - start
+ break
+ else:
+ count = count + 1
+ cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip)
- cli_result1 = main.ONOScli1.start_onos_cli(ONOS1_ip)
-
- case_results = main.TRUE and onosi1_isup and cli_result1
+ case_results = main.TRUE and onosi1_isup and cli_result
utilities.assert_equals(expect=main.TRUE, actual=case_results,
onpass="ONOS restart successful",
onfail="ONOS restart NOT successful")
+ main.log.info("ONOS took %s seconds to restart" % str(elapsed) )
def CASE7(self,main) :
'''
@@ -451,9 +462,10 @@
main.step("Check if switch roles are consistent across all nodes")
ONOS1_mastership = main.ONOScli1.roles()
+ #FIXME: Refactor this whole case for single instance
#print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': '))
if "Error" in ONOS1_mastership or not ONOS1_mastership:
- main.log.error("Error in getting ONOS mastership")
+ main.log.report("Error in getting ONOS mastership")
main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership))
consistent_mastership = main.FALSE
else:
@@ -589,18 +601,6 @@
ctrls.append(temp)
MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations
- main.step("Collecting topology information from ONOS")
- devices = []
- devices.append( main.ONOScli1.devices() )
- '''
- hosts = []
- hosts.append( main.ONOScli1.hosts() )
- '''
- ports = []
- ports.append( main.ONOScli1.ports() )
- links = []
- links.append( main.ONOScli1.links() )
-
main.step("Comparing ONOS topology to MN")
devices_results = main.TRUE
ports_results = main.TRUE
@@ -608,8 +608,21 @@
topo_result = main.FALSE
start_time = time.time()
elapsed = 0
+ count = 0
while topo_result == main.FALSE and elapsed < 120:
+ count = count + 1
try:
+ main.step("Collecting topology information from ONOS")
+ devices = []
+ devices.append( main.ONOScli1.devices() )
+ '''
+ hosts = []
+ hosts.append( main.ONOScli1.hosts() )
+ '''
+ ports = []
+ ports.append( main.ONOScli1.ports() )
+ links = []
+ links.append( main.ONOScli1.links() )
for controller in range(1): #TODO parameterize the number of controllers
if devices[controller] or not "Error" in devices[controller]:
current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller]))
@@ -643,10 +656,12 @@
devices_results = devices_results and current_devices_result
ports_results = ports_results and current_ports_result
links_results = links_results and current_links_result
- topo_result = devices_results and ports_results and links_results
- elapsed = time.time()-start_time()
+ elapsed = time.time() - start_time
+ time_threshold = elapsed < 1
+ topo_result = devices_results and ports_results and links_results and time_threshold
#TODO make sure this step is non-blocking. IE add a timeout
- main.log.report("Very crass estimate for topology discovery/convergence: " + str(elapsed) + " seconds")
+ main.log.report("Very crass estimate for topology discovery/convergence: " +\
+ str(elapsed) + " seconds, " + str(count) +" tries" )
utilities.assert_equals(expect=main.TRUE, actual=topo_result,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful")