make image cleanup more robust; add check/start docker case; add check onos log; other minor fixes
Change-Id: I47c8d79d01cb7353d60ab340863034f2891c1663
diff --git a/TestON/drivers/common/api/dockerapidriver.py b/TestON/drivers/common/api/dockerapidriver.py
index 21809aa..0444665 100644
--- a/TestON/drivers/common/api/dockerapidriver.py
+++ b/TestON/drivers/common/api/dockerapidriver.py
@@ -49,7 +49,8 @@
": Pulling Docker image " + onosRepo + ":"+ onosTag )
for line in self.dockerClient.pull( repository = onosRepo, \
tag = onosTag, stream = True ):
- print(json.dumps(json.loads(line), indent =4))
+ print "#",
+ main.log.info(json.dumps(json.loads(line), indent =4))
#response = json.dumps( json.load( pullResult ), indent=4 )
if re.search( "for onosproject/onos:latest", line ):
@@ -58,7 +59,7 @@
else:
main.log.error( "Failed to download image from: " + onosRepo +":"+ onosTag )
main.log.error( "Error respone: " )
- main.log.error( response )
+ main.log.error( line )
return main.FALSE
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
@@ -198,27 +199,32 @@
"""
Remove Docker image
"""
+ rmResult = main.TRUE
+
if self.dockerClient.images() is []:
main.log.info( "No docker image found with RepoTags: " + image )
- return main.TRUE
+ return rmResult
else:
- try:
- main.log.info( self.name +
- ": Removing Docker image " + image )
- response = self.dockerClient.remove_image( image, force = True)
- if response is None:
- main.log.info( "Removed Docker image: " + image )
- return main.TRUE
- else:
- main.log.info( "Noticed warnings during Remove " + image )
- return main.FALSE
- except errors.NotFound:
- main.log.warn( image + "not found! Continue with the tests...")
- return main.TRUE
- except Exception:
- main.log.exception( self.name + ": Uncaught exception! Continuing..." )
- #main.cleanup()
- #main.exit()
+ list = [ image["Id"] for image in self.dockerClient.images() if image["RepoTags"][0] == '<none>:<none>' ]
+ for id in list:
+ try:
+ main.log.info( self.name + ": Removing Docker image " + id )
+ response = self.dockerClient.remove_image(id, force = True)
+ if response is None:
+ main.log.info( "Removed Docker image: " + id )
+ rmResult = rmResult and main.TRUE
+ else:
+ main.log.info( "Noticed warnings during Remove " + id )
+ rmResult = rmResult and main.FALSE
+ except errors.NotFound:
+ main.log.warn( image + "not found! Continue with the tests...")
+ rmResult = rmResult and main.TRUE
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception! Continuing..." )
+ rmResult = rmResult and main.FALSE
+ #main.cleanup()
+ #main.exit()
+ return rmResult
def fetchLatestClusterFile( self, branch="master" ):
"""
diff --git a/TestON/tests/PLATdockertest/PLATdockertest.params b/TestON/tests/PLATdockertest/PLATdockertest.params
index 105be40..b16be02 100755
--- a/TestON/tests/PLATdockertest/PLATdockertest.params
+++ b/TestON/tests/PLATdockertest/PLATdockertest.params
@@ -1,6 +1,6 @@
<PARAMS>
- <testcases>1,1000,5,10,110,1000</testcases>
+ <testcases>1,1000,5,10,110,900,1000</testcases>
<DOCKER>
<repo>onosproject/onos</repo>
@@ -14,7 +14,7 @@
</SCALE>
<CASE110>
- <clustercmdpath>~/OnosSystemTest/TestON/tests/PLATdockertest/Dependency</clustercmdpath>
+ <clustercmdpath>../tests/PLATdockertest/Dependency</clustercmdpath>
<apps>org.onosproject.proxyarp,org.onosproject.fwd</apps>
</CASE110>
diff --git a/TestON/tests/PLATdockertest/PLATdockertest.py b/TestON/tests/PLATdockertest/PLATdockertest.py
old mode 100644
new mode 100755
index 629771d..7531511
--- a/TestON/tests/PLATdockertest/PLATdockertest.py
+++ b/TestON/tests/PLATdockertest/PLATdockertest.py
@@ -29,6 +29,8 @@
1) set up test params;
"""
import re
+ import time
+ import subprocess
main.case("Set case test params")
main.step("Initialize test params")
@@ -39,9 +41,23 @@
DOCKERREPO = main.params[ 'DOCKER' ][ 'repo' ]
DOCKERTAG = main.params[ 'DOCKER' ][ 'tag' ]
CTIDlist = list()
- utilities.assert_equals( expect = main.TRUE, actual = main.TRUE,
- onpass = "Params set",
- onfail = "Failed to set params")
+
+ main.log.info("Check docker status, it not running, try restart it")
+ iter = 0
+ stepResult = main.TRUE
+ while subprocess.call("sudo service docker status", shell=True) and iter <= 3:
+ subprocess.call("sudo service docker restart", shell=True)
+ time.sleep(5)
+ iter += 1
+ if iter == 3: stepResult = main.FALSE
+
+ utilities.assert_equals( expect = main.TRUE, actual = stepResult,
+ onpass = "docker is running",
+ onfail = "docker is not running")
+ if stepResult == main.FALSE:
+ main.log.warn("docker is not running - exiting test")
+ main.exit()
+ main.cleanup()
def CASE5(self, main):
"""
@@ -56,13 +72,15 @@
utilities.assert_equals( expect = main.TRUE, actual = stepResult,
onpass = "Succeeded in pulling " + DOCKERREPO + ":" + DOCKERTAG,
onfail = "Failed to pull " + DOCKERREPO + ":" + DOCKERTAG )
+ if stepResult == main.FALSE: main.skipCase()
def CASE10( self, main ):
"""
Start docker containers for list of onos nodes, only if not already existed
"""
import re
-
+ createResult = main.TRUE
+ startResult = main.TRUE
main.case( "Start onos container(s)")
image = DOCKERREPO + ":" + DOCKERTAG
@@ -134,7 +152,7 @@
utilities.assert_equals( expect = main.TRUE, actual = stepResult,
onpass = "ONOS successfully started",
onfail = "Failed to start ONOS correctly" )
- if stepResult is main.FALSE: main.skipCase
+ if stepResult is main.FALSE: main.skipCase()
main.step( "Form onos cluster using 'Dependency/onos-form-cluster' util")
stepResult = main.FALSE
@@ -150,12 +168,14 @@
if status == 200:
jrsp = json.loads(response)
clusterIP = [item["ip"]for item in jrsp["nodes"] if item["status"]== "ACTIVE"]
+ main.log.debug(" IPlist is:" + ",".join(IPlist))
+ main.log.debug("cluster IP is" + ",".join(clusterIP) )
if set(IPlist) == set(clusterIP): stepResult = main.TRUE
utilities.assert_equals( expect = main.TRUE, actual = stepResult,
onpass = "ONOS successfully started",
onfail = "Failed to start ONOS correctly" )
- if stepResult is main.FALSE: main.skipCase
+ if stepResult is main.FALSE: main.skipCase()
main.step( "Check cluster app status")
stepResult = main.TRUE
@@ -166,15 +186,15 @@
main.log.info("Some bundles are not in correct state. ")
main.log.info("App states are: " + response)
stepResult = main.FALSE
- break;
+ break
if (item["description"] == "Builtin device drivers") and (item["state"] != "ACTIVE"):
main.log.info("Driver app is not in 'ACTIVE' state, but in: " + item["state"])
stepResult = main.FALSE
- break;
+ break
utilities.assert_equals( expect = main.TRUE, actual = stepResult,
onpass = "ONOS successfully started",
onfail = "Failed to start ONOS correctly" )
- if stepResult is main.FALSE: main.skipCase
+ if stepResult is main.FALSE: main.skipCase()
main.step(" Activate an APP from REST and check APP status")
appResults = list()
@@ -190,7 +210,7 @@
utilities.assert_equals( expect = main.TRUE, actual = stepResult,
onpass = "Successfully activated apps",
onfail = "Failed to activated apps correctly" )
- if stepResult is main.FALSE: main.skipCase
+ if stepResult is main.FALSE: main.skipCase()
main.step(" Deactivate an APP from REST and check APP status")
appResults = list()
@@ -206,7 +226,51 @@
utilities.assert_equals( expect = main.TRUE, actual = stepResult,
onpass = "Successfully deactivated apps",
onfail = "Failed to deactivated apps correctly" )
- if stepResult is main.FALSE: main.skipCase
+ if stepResult is main.FALSE: main.skipCase()
+
+ def CASE900(self,main):
+ """
+ Check onos logs for exceptions after tests
+ """
+ import pexpect
+ import time
+ import re
+
+ logResult = main.TRUE
+
+ user = main.params["DOCKER"]["user"]
+ pwd = main.params["DOCKER"]["password"]
+
+ main.case("onos Exceptions check")
+ main.step("check onos for any exceptions")
+
+ for ip in IPlist:
+ spawncmd = "ssh -p 8101 " + user + "@" + ip
+ main.log.info("log on node using cmd: " + spawncmd)
+ try:
+ handle = pexpect.spawn(spawncmd)
+ handle.expect("yes/no")
+ handle.sendline("yes")
+ handle.expect("Password:")
+ handle.sendline(pwd)
+ time.sleep(5)
+ handle.expect("onos>")
+ handle.sendline("log:exception-display")
+ handle.expect("onos>")
+ result = handle.before
+ if re.search("Exception", result):
+ main.log.info("onos: " + ip + " Exceptions:" + result)
+ logResult = logResult and main.FALSE
+ else:
+ main.log.info("onos: " + ip + " Exceptions: None")
+ logResult = logResult and main.TRUE
+ except Exception:
+ main.log.exception("Uncaught exception when getting log from onos:" + ip)
+ logResult = logResult and main.FALSE
+
+ utilities.assert_equals( expect = main.TRUE, actual = logResult,
+ onpass = "onos exception check passed",
+ onfail = "onos exeption check failed" )
def CASE1000( self, main ):
@@ -215,7 +279,7 @@
"""
import time
- main.case("Clean up unwanted images and containers")
+ main.case("Clean up images (ex. none:none tagged) and containers")
main.step("Stop onos containers")
stepResult = main.TRUE
for ctname in NODElist:
@@ -231,9 +295,9 @@
#main.step( "remove exiting onosproject/onos images")
#stepResult = main.ONOSbenchDocker.dockerRemoveImage( image = DOCKERREPO + ":" + DOCKERTAG )
- main.step( "remove exiting 'none:none' images")
+ main.step( "remove dangling 'none:none' images")
stepResult = main.ONOSbenchDocker.dockerRemoveImage( image = "<none>:<none>" )
utilities.assert_equals( expect = main.TRUE, actual = stepResult,
- onpass = "Succeeded in deleting image " + "<none>:<none>",
- onfail = "Failed to delete image " + "<none>:<none>" )
+ onpass = "Succeeded in cleaning up images",
+ onfail = "Failed in cleaning up images" )