Save ONOS Logs for cluster test, print stacktrace when looking for
exceptions, and other random fixes
Change-Id: I26d54fc261d91c88b2303e3d99a954597fc4c58c
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 2b37186..6c36394 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -711,6 +711,7 @@
"=" + "\"" + arg + "\"" + "\n" )
tempCount = tempCount + 1
+ cellFile.write( "export OCI=$OC1" )
cellFile.write( mnString + "\"" + mnIpAddrs + "\"" + "\n" )
cellFile.write( appString + "\n" )
cellFile.close()
@@ -1435,9 +1436,9 @@
self.handle.expect( "\$" )
self.handle.sendline( "tshark -i " + str( interface ) + " -t e -w " + str( dirFile ) + " &" )
- self.handle.sendline( "\r" )
+ self.handle.sendline( "\n" )
self.handle.expect( "Capturing on" )
- self.handle.sendline( "\r" )
+ self.handle.sendline( "\n" )
self.handle.expect( "\$" )
main.log.info( "Tshark started capturing files on " +
@@ -1661,7 +1662,7 @@
cmd += " old"
self.handle.sendline( cmd )
self.handle.expect( cmd )
- self.handle.expect( "\$" )
+ self.handle.expect( "\$ " )
response = self.handle.before
return response
except pexpect.EOF:
@@ -1730,7 +1731,6 @@
* This function uses root privilege iptables command which may result
in unwanted network errors. USE WITH CAUTION
"""
- import time
# NOTE*********
# The strict checking methods of this driver function is intentional
diff --git a/TestON/drivers/common/clidriver.py b/TestON/drivers/common/clidriver.py
index fd1a401..b6be81a 100644
--- a/TestON/drivers/common/clidriver.py
+++ b/TestON/drivers/common/clidriver.py
@@ -232,7 +232,7 @@
i = handle.expect( [ ".ssword:*", default, pexpect.EOF ] )
if i == 0:
handle.sendline( pwd )
- handle.sendline( "\r" )
+ handle.sendline( "\n" )
if i == 1:
handle.expect( default )
diff --git a/TestON/tests/HAclusterRestart/HAclusterRestart.params b/TestON/tests/HAclusterRestart/HAclusterRestart.params
index 627bec2..0f37220 100644
--- a/TestON/tests/HAclusterRestart/HAclusterRestart.params
+++ b/TestON/tests/HAclusterRestart/HAclusterRestart.params
@@ -40,7 +40,7 @@
<port7>6633</port7>
</CTRL>
<BACKUP>
- <ENABLED> False </ENABLED>
+ <ENABLED>True</ENABLED>
<TESTONUSER>admin</TESTONUSER>
<TESTONIP>10.128.30.9</TESTONIP>
</BACKUP>
diff --git a/TestON/tests/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HAclusterRestart/HAclusterRestart.py
index b5ebe7d..f7eceb3 100644
--- a/TestON/tests/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HAclusterRestart/HAclusterRestart.py
@@ -1717,6 +1717,11 @@
main.case( "Restart entire ONOS cluster" )
+ main.step( "Checking ONOS Logs for errors" )
+ for node in main.nodes:
+ main.log.debug( "Checking logs for errors on " + node.name + ":" )
+ main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
+
main.step( "Killing ONOS nodes" )
killResults = main.TRUE
killTime = time.time()
@@ -2694,7 +2699,8 @@
main.Mininet2.stopTcpdump()
testname = main.TEST
- if main.params[ 'BACKUP' ] == "True":
+ main.log.error( main.params[ 'BACKUP' ][ 'ENABLED' ] )
+ if main.params[ 'BACKUP' ][ 'ENABLED' ] == "True":
main.step( "Copying MN pcap and ONOS log files to test station" )
teststationUser = main.params[ 'BACKUP' ][ 'TESTONUSER' ]
teststationIP = main.params[ 'BACKUP' ][ 'TESTONIP' ]
@@ -2707,36 +2713,38 @@
logFolder = "/opt/onos/log/"
logFiles = [ "karaf.log", "karaf.log.1" ]
# NOTE: must end in /
- dstDir = "~/packet_captures/"
for f in logFiles:
for node in main.nodes:
main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
":" + logFolder + f + " " +
teststationUser + "@" +
teststationIP + ":" +
- dstDir + str( testname ) +
- "-" + node.name + "-" + f )
- main.ONOSbench.handle.expect( "\$" )
+ main.logdir + "/" +
+ node.name + "-" + f )
+ main.ONOSbench.handle.expect( "\$ " )
+ print main.ONOSbench.handle.before
# std*.log's
# NOTE: must end in /
logFolder = "/opt/onos/var/"
logFiles = [ "stderr.log", "stdout.log" ]
# NOTE: must end in /
- dstDir = "~/packet_captures/"
for f in logFiles:
for node in main.nodes:
main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
":" + logFolder + f + " " +
teststationUser + "@" +
teststationIP + ":" +
- dstDir + str( testname ) +
- "-" + node.name + "-" + f )
- main.ONOSbench.handle.expect( "\$" )
+ main.logdir + "/" +
+ node.name + "-" + f )
+ main.ONOSbench.handle.expect( "\$ " )
+ print main.ONOSbench.handle.before
# sleep so scp can finish
time.sleep( 10 )
- main.step( "Packing and rotating pcap archives" )
- os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+ # main.step( "Packing and rotating pcap archives" )
+ # os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+ else:
+ main.log.debug( "skipping saving log files" )
main.step( "Stopping Mininet" )
mnResult = main.Mininet1.stopNet()
@@ -2746,9 +2754,8 @@
main.step( "Checking ONOS Logs for errors" )
for node in main.nodes:
- print colors[ 'purple' ] + "Checking logs for errors on " + \
- node.name + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( node.ip_address, restart=True )
+ main.log.debug( "Checking logs for errors on " + node.name + ":" )
+ main.log.warn( main.ONOSbench.checkLogs( node.ip_address ) )
try:
timerLog = open( main.logdir + "/Timers.csv", 'w')