Changes to HA tests
* add tcpdump
* add csv output for pass/fail # to be used by jenkins plot plugin
* remove mininet commands from remotemininetdriver
* fix some typos
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 44a70b8..9c5ad4d 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -42,7 +42,7 @@
logmsg = logmsg + "\n\tTest Script :" + path + "Tests/" + main.TEST + ".py"+ ""
logmsg = logmsg + "\n\tTest Params : " + path + "Tests/" + main.TEST + ".params" + ""
- logmsg = logmsg + "\n\tTopology : " + path + "Tests/" +main.TEST + ".tpl" + ""
+ logmsg = logmsg + "\n\tTopology : " + path + "Tests/" +main.TEST + ".topo" + ""
logmsg = logmsg + "\n"+" " * 30+"+" +"-" * 18+"+" +"\n" +"-" * 27+" { Script Exec Params } "+"-" * 27 +"\n" +" " * 30 +"+"+"-" * 18 +"+\n";
values = "\n\t" + str(main.params)
values = re.sub(",", "\n\t", values)
@@ -110,6 +110,7 @@
main.LogFileName = main.logdir + "/" + main.TEST + "_" +str(currentTime) + ".log"
main.ReportFileName = main.logdir + "/" + main.TEST + "_" + str(currentTime) + ".rpt"
+ main.JenkinsCSV = main.logdir + "/" + main.TEST + ".csv"
#### Add log-level - Report
logging.addLevelName(9, "REPORT")
@@ -228,7 +229,17 @@
#main.log.report(testResult)
main.testResult = testResult
main.log.exact(testResult)
-
+
+ ##CSV output needed for Jenkin's plot plugin
+ #NOTE: the elements were orded based on the colors assigned to the data
+ logfile = open(main.JenkinsCSV ,"w")
+ logfile.write(",".join( ['Tests Failed', 'Tests Passed', 'Tests Planned'] ) + "\n")
+ logfile.write(",".join( [str(main.TOTAL_TC_FAIL), str(main.TOTAL_TC_PASS), str(main.TOTAL_TC_PLANNED)] ))
+ logfile.close()
+
+
+
+
def updateCaseResults(self,main):
'''
Update the case result based on the steps execution and asserting each step in the test-case
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index bb03c07..9cf13ed 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -80,9 +80,16 @@
Returns main.ERROR if "found multiple mininet" is found and
Returns main.TRUE else
'''
+ #TODO: maybe we want to return the % loss instead? This way we can set an acceptible loss %.
+ #EX: 393 packets transmitted, 380 received, 3% packet loss, time 78519ms
+ # we may need to return a float to get around rounding errors
+
import os
self.handle.sendline("")
self.handle.expect("\$")
+ #Clear any output waiting in the bg from killing pings
+ self.handle.sendline("")
+ self.handle.expect("\$")
self.handle.sendline("cat " + pingList)
self.handle.expect(pingList)
self.handle.expect("\$")
@@ -156,8 +163,13 @@
command = "scp /tmp/ping.* "+ str(testONUser) + "@" + str(testONIP) + ":/tmp/"
self.execute(cmd=command,prompt="100%",timeout=20)
self.handle.sendline("")
- self.handle.expect("\$")
- return main.TRUE
+ i=self.handle.expect(["password","\$"])
+ if i == 0:
+ main.log.error("Error, sudo asking for password")
+ main.log.error(self.handle.before)
+ return main.FALSE
+ else:
+ return main.TRUE
def pingLongKill(self):
import time
@@ -209,34 +221,6 @@
main.last_result = main.FALSE
return main.FALSE
-
- def pingall(self):
- '''
- Verifies the reachability of the hosts using pingall command.
- This function is required by Packey Optical test
- '''
- if self.handle :
- main.log.info(self.name+": Checking reachabilty to the hosts using pingall")
- try:
- response = self.execute(cmd="pingall",prompt="mininet>",timeout=120)
- print "response: " + str(response)
- except pexpect.EOF:
- main.log.error(self.name + ": EOF exception found")
- main.log.error(self.name + ": " + self.handle.before)
- pattern = 'Results\:\s0\%\sdropped\s'
- if re.search(pattern,response):
- main.log.info(self.name+": All hosts are reachable")
- return main.TRUE
- else:
- main.log.error(self.name+": Unable to reach all the hosts")
- return main.FALSE
- else :
- main.log.error(self.name+": Connection failed to the host")
- return main.FALSE
-
-
-
-
def pingHost(self,**pingParams):
'''
Pings between two hosts on remote mininet
@@ -286,146 +270,6 @@
else :
main.log.error("Connection failed to the host")
- def ctrl_none(self):
- '''
- Sets all the switches to no controllers.
- '''
- self.execute(cmd="~/ONOS/scripts/test-ctrl-none.sh", prompt="\$",timeout=10)
-
- def ctrl_one(self, ip):
- '''
- Sets all the switches to point to the supplied IP
- '''
- self.execute(cmd="~/ONOS/scripts/test-ctrl-one.sh "+ip, prompt="\$",timeout=10)
-
- def ctrl_local(self):
- '''
- Sets all the switches to point to the Controller on the same machine that they are running on.
- '''
- self.execute(cmd="~/ONOS/scripts/test-ctrl-local.sh ", prompt="\$",timeout=10)
-
- # def verifySSH(self,**connectargs):
- # response = self.execute(cmd="h1 /usr/sbin/sshd -D&",prompt="mininet>",timeout=10)
- # response = self.execute(cmd="h4 /usr/sbin/sshd -D&",prompt="mininet>",timeout=10)
- # for key in connectargs:
- # vars(self)[key] = connectargs[key]
- # response = self.execute(cmd="xterm h1 h4 ",prompt="mininet>",timeout=10)
- # import time
- # time.sleep(20)
- # if self.flag == 0:
- # self.flag = 1
- # return main.FALSE
- # else :
- # return main.TRUE
- #
- # def getMacAddress(self,host):
- # '''
- # Verifies the host's ip configured or not.
- # '''
- # if self.handle :
- # response = self.execute(cmd=host+" ifconfig",prompt="mininet>",timeout=10)
-
- # pattern = "HWaddr\s(((\d|\w)+:)+(\d|\w))"
- # mac_address_search = re.search(pattern, response)
- # main.log.info("Mac-Address of Host "+host +" is "+mac_address_search.group(1))
- # return mac_address_search.group(1)
- # else :
- # main.log.error("Connection failed to the host")
- # def getIPAddress(self,host):
- # '''
- # Verifies the host's ip configured or not.
- # '''
- # if self.handle :
- # response = self.execute(cmd=host+" ifconfig",prompt="mininet>",timeout=10)
-
- # pattern = "inet\saddr:(\d+\.\d+\.\d+\.\d+)"
- # ip_address_search = re.search(pattern, response)
- # main.log.info("IP-Address of Host "+host +" is "+ip_address_search.group(1))
- # return ip_address_search.group(1)
- # else :
- # main.log.error("Connection failed to the host")
- #
- # def dump(self):
- # main.log.info("Dump node info")
- # self.execute(cmd = 'dump',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def intfs(self):
- # main.log.info("List interfaces")
- # self.execute(cmd = 'intfs',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def net(self):
- # main.log.info("List network connections")
- # self.execute(cmd = 'net',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def iperf(self):
- # main.log.info("Simple iperf TCP test between two (optionally specified) hosts")
- # self.execute(cmd = 'iperf',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def iperfudp(self):
- # main.log.info("Simple iperf TCP test between two (optionally specified) hosts")
- # self.execute(cmd = 'iperfudp',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def nodes(self):
- # main.log.info("List all nodes.")
- # self.execute(cmd = 'nodes',prompt = 'mininet>',timeout = 10)
- # return main.TRUE
- #
- # def pingpair(self):
- # main.log.infoe("Ping between first two hosts")
- # self.execute(cmd = 'pingpair',prompt = 'mininet>',timeout = 20)
- #
- # if utilities.assert_matches(expect='0% packet loss',actual=response,onpass="No Packet loss",onfail="Hosts not reachable"):
- # main.log.info("Ping between two hosts SUCCESS")
- # main.last_result = main.TRUE
- # return main.TRUE
- # else :
- # main.log.error("PACKET LOST, HOSTS NOT REACHABLE")
- # main.last_result = main.FALSE
- # return main.FALSE
- #
- # def link(self,**linkargs):
- # '''
- # Bring link(s) between two nodes up or down
- # '''
- # main.log.info('Bring link(s) between two nodes up or down')
- # args = utilities.parse_args(["END1","END2","OPTION"],**linkargs)
- # end1 = args["END1"] if args["END1"] != None else ""
- # end2 = args["END2"] if args["END2"] != None else ""
- # option = args["OPTION"] if args["OPTION"] != None else ""
- # command = "link "+str(end1) + " " + str(end2)+ " " + str(option)
- # response = self.execute(cmd=command,prompt="mininet>",timeout=10)
- # return main.TRUE
- #
-
- # def dpctl(self,**dpctlargs):
- # '''
- # Run dpctl command on all switches.
- # '''
- # main.log.info('Run dpctl command on all switches')
- # args = utilities.parse_args(["CMD","ARGS"],**dpctlargs)
- # cmd = args["CMD"] if args["CMD"] != None else ""
- # cmdargs = args["ARGS"] if args["ARGS"] != None else ""
- # command = "dpctl "+cmd + " " + str(cmdargs)
- # response = self.execute(cmd=command,prompt="mininet>",timeout=10)
- # return main.TRUE
- #
- #
- # def get_version(self):
- # file_input = path+'/lib/Mininet/INSTALL'
- # version = super(Mininet, self).get_version()
- # pattern = 'Mininet\s\w\.\w\.\w\w*'
- # for line in open(file_input,'r').readlines():
- # result = re.match(pattern, line)
- # if result:
- # version = result.group(0)
- #
- #
- # return version
def start_tcpdump(self, filename, intf = "eth0", port = "port 6633"):
'''
Runs tpdump on an intferface and saves the file
@@ -502,49 +346,6 @@
main.log.error(self.name + ": EOF exception found")
main.log.error(self.name + ": " + self.handle.before)
return main.FALSE
-
- def del_switch(self,sw):
- self.handle.sendline("")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl del-br "+sw)
- self.handle.expect("\$")
- return main.TRUE
-
- def add_switch(self,sw):
- #FIXME: Remove hardcoded number of ports
- self.handle.sendline("")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-br "+sw)
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth1")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth2")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth3")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth4")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth5")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth6")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth7")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth8")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth9")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth10")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth11")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth12")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth13")
- self.handle.expect("\$")
- self.handle.sendline("sudo ovs-vsctl add-port "+sw+" " + sw + "-eth14")
- self.handle.expect("\$")
-
def disconnect(self):
'''
@@ -565,9 +366,7 @@
def get_flowTable(self, protoVersion, sw):
self.handle.sendline("cd")
- #self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
- print "cd expect status: "
- print self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
+ self.handle.expect(["\$",pexpect.EOF,pexpect.TIMEOUT])
#TODO: Write seperate versions of the function for this, possibly a string that tells it which switch is in use?
#For 1.0 version of OVS
#command = "sudo ovs-ofctl dump-flows " + sw + " | awk '{OFS=\",\" ; print $1 $6 $7 }' |sort -n -k1"
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 19665a4..35a5d9d 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -395,7 +395,7 @@
self.handle.sendline("\n")
self.handle.expect("\$")
self.handle.sendline("cd " + self.home + "; git log -1 --pretty=fuller --decorate=short | grep -A 6 \"commit\" --color=never")
- self.handle.expect("--color=never")
+ #self.handle.expect("--color=never")
self.handle.expect("\$")
response=(self.name +": \n"+ str(self.handle.before + self.handle.after))
self.handle.sendline("cd " + self.home)
@@ -417,6 +417,11 @@
main.log.error(self.name + ": " + self.handle.before)
main.cleanup()
main.exit()
+ except pexpect.TIMEOUT:
+ main.log.error(self.name + ": TIMEOUT exception found")
+ main.log.error(self.name + ": " + self.handle.before)
+ main.cleanup()
+ main.exit()
except:
main.log.info(self.name + ":::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::")
main.log.error( traceback.print_exc() )
@@ -1231,5 +1236,3 @@
main.log.error( traceback.print_exc())
main.log.info(self.name+" ::::::")
-
-
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index 9ae90fb..e54c663 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -142,7 +142,7 @@
main.log.info("Found More screen to go , Sending a key to proceed")
indexMore = self.handle.expect(["^:$", expectPrompt], timeout = timeoutVar)
while indexMore == 0:
- main.log.info("Found anoother More screen to go , Sending a key to proceed")
+ main.log.info("Found another More screen to go , Sending a key to proceed")
self.handle.sendcontrol("D")
indexMore = self.handle.expect(["^:$", expectPrompt], timeout = timeoutVar)
self.LASTRSP = self.LASTRSP + self.handle.before
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index dad0fd7..e1e75f9 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -1,5 +1,5 @@
<PARAMS>
- <testcases>1,2,8,3,4,5,6,7,8,9,8,10,8,11,8,12,8,13</testcases>
+ <testcases>1,2,8,3,4,5,[6],7,8,4,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 8936578..03fcfe7 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -10,7 +10,7 @@
CASE4: Ping across added host intents
CASE5: Reading state of ONOS
CASE6: The Failure case. Since this is the Sanity test, we do nothing.
-CASE7: Check state after failure
+CASE7: Check state after control plane failure
CASE8: Compare topo
CASE9: Link s3-s28 down
CASE10: Link s3-s28 up
@@ -38,7 +38,7 @@
onos-wait-for-start
'''
import time
- main.log.report("ONOS HA Sanity test initialization")
+ main.log.report("ONOS HA Sanity test - initialization")
main.case("Setting up test environment")
# load some vairables from the params file
@@ -82,6 +82,14 @@
main.step("Applying cell variable to environment")
cell_result = main.ONOSbench.set_cell(cell_name)
verify_result = main.ONOSbench.verify_cell()
+
+ main.ONOSbench.onos_stop(ONOS1_ip)
+ main.ONOSbench.onos_stop(ONOS2_ip)
+ main.ONOSbench.onos_stop(ONOS3_ip)
+ main.ONOSbench.onos_stop(ONOS4_ip)
+ main.ONOSbench.onos_stop(ONOS5_ip)
+ main.ONOSbench.onos_stop(ONOS6_ip)
+ main.ONOSbench.onos_stop(ONOS7_ip)
#FIXME:this is short term fix
main.ONOSbench.onos_remove_raft_logs()
@@ -99,7 +107,7 @@
if git_pull_result == main.TRUE:
clean_install_result = main.ONOSbench.clean_install()
else:
- main.log.report("Did not pull new code so skipping mvn "+ \
+ main.log.warn("Did not pull new code so skipping mvn "+ \
"clean install")
main.ONOSbench.get_version(report=True)
@@ -148,12 +156,11 @@
main.ONOScli6.start_onos_cli(ONOS6_ip)
main.ONOScli7.start_onos_cli(ONOS7_ip)
- #TODO: Enable once test is ready
- #main.step("Start Packet Capture MN")
- #main.Mininet2.start_tcpdump(
- # str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
- # intf = main.params['MNtcpdump']['intf'],
- # port = main.params['MNtcpdump']['port'])
+ main.step("Start Packet Capture MN")
+ main.Mininet2.start_tcpdump(
+ str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap",
+ intf = main.params['MNtcpdump']['intf'],
+ port = main.params['MNtcpdump']['port'])
case1_result = (clean_install_result and package_result and
@@ -289,6 +296,9 @@
"""
Ping across added host intents
"""
+ description = " Ping across added host intents"
+ main.log.report(description)
+ main.case(description)
Ping_Result = main.TRUE
for i in range(8,18):
ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10))
@@ -301,9 +311,10 @@
if Ping_Result==main.FALSE:
main.log.report("Intents have not been installed correctly, pings failed.")
if Ping_Result==main.TRUE:
- main.log.report("Intents have been installed correctly, and verified by pings")
+ main.log.report("Intents have been installed correctly and verified by pings")
utilities.assert_equals(expect = main.TRUE,actual=Ping_Result,
- onpass="Intents have been installed correctly and pings work")
+ onpass="Intents have been installed correctly and pings work",
+ onfail ="Intents have not been installed correctly, pings failed." )
def CASE5(self,main) :
'''
@@ -592,10 +603,15 @@
'''
The Failure case. Since this is the Sanity test, we do nothing.
'''
+ import time
+ time.sleep(60)
+ utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+ onpass="Sleeping 60 seconds",
+ onfail="Something is terribly wrong with my math")
def CASE7(self,main) :
'''
- Check state after failure
+ Check state after ONOS failure
'''
import os
import json
@@ -643,8 +659,6 @@
description2 = "Compare switch roles from before failure"
main.step(description2)
-
-
current_json = json.loads(ONOS1_mastership)
old_json = json.loads(mastership_state)
mastership_check = main.TRUE
@@ -982,6 +996,9 @@
'''
Clean up
'''
+ description = "Test Cleanup"
+ main.log.report(description)
+ main.case(description)
main.step("Killing tcpdumps")
main.Mininet2.stop_tcpdump()
@@ -1017,3 +1034,8 @@
main.ONOSbench.onos_stop(ONOS5_ip)
main.ONOSbench.onos_stop(ONOS6_ip)
main.ONOSbench.onos_stop(ONOS7_ip)
+
+ #TODO: actually check something here
+ utilities.assert_equals(expect=main.TRUE, actual=main.TRUE,
+ onpass="Test cleanup successful",
+ onfail="Test cleanup NOT successful")