Merge branch 'master' of https://github.com/opennetworkinglab/ONLabTest into devl/newFunctionality
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index 8187e4e..fc85c55 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -288,13 +288,12 @@
self.stepCache += "No Result\n"
#self.stepCache += "No Result <ac:emoticon ac:name=\"warning\" /></li>\n"
self.stepResults.append(self.STEPRESULT)
- except TypeError,e:
- print "\nException in the following section of code: " +\
- str(testCaseNumber) + "." + str(step) + ": " +\
- self.stepName
+ except StandardError as e:
+ self.log.exception( "\nException in the following section of" +
+ " code: " + str(testCaseNumber) + "." +
+ str(step) + ": " + self.stepName )
#print code[testCaseNumber][step]
self.stepCount = self.stepCount + 1
- self.log.exception(e)
self.logger.updateCaseResults(self)
#WIKI results
self.log.wiki( "<ul>" )
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index 0004cfc..3b0b01b 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -91,8 +91,9 @@
elif re.search( "found multiple mininet", outputs ):
return main.ERROR
else:
+ # TODO: Parse for failed pings, give some truncated output
main.log.error( "Error, unexpected output in the ping file" )
- #main.log.warn( outputs )
+ main.log.warn( outputs )
return main.TRUE
def arping( self, host="", ip="10.128.20.211" ):
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index b6f931e..1d57271 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -188,7 +188,8 @@
main.cleanup()
main.exit()
- def startOnosCli( self, ONOSIp, karafTimeout="" ):
+ def startOnosCli( self, ONOSIp, karafTimeout="",
+ commandlineTimeout=10, onosStartTimeout=60 ):
"""
karafTimeout is an optional argument. karafTimeout value passed
by user would be used to set the current karaf shell idle timeout.
@@ -206,7 +207,7 @@
try:
self.handle.sendline( "" )
x = self.handle.expect( [
- "\$", "onos>" ], timeout=10 )
+ "\$", "onos>" ], commandlineTimeout)
if x == 1:
main.log.info( "ONOS cli is already running" )
@@ -216,7 +217,7 @@
self.handle.sendline( "onos -w " + str( ONOSIp ) )
i = self.handle.expect( [
"onos>",
- pexpect.TIMEOUT ], timeout=60 )
+ pexpect.TIMEOUT ], onosStartTimeout )
if i == 0:
main.log.info( str( ONOSIp ) + " CLI Started successfully" )
@@ -280,7 +281,11 @@
lvlStr = "--level=" + level
self.handle.sendline( "" )
- self.handle.expect( "onos>" )
+ i = self.handle.expect( [ "onos>", pexpect.TIMEOUT ] )
+ # TODO: look for bash prompt as well
+ if i == 1:
+ self.handle.sendline( "" )
+ self.handle.expect( "onos>" )
self.handle.sendline( "log:log " + lvlStr + " " + cmdStr )
self.handle.expect( "log:log" )
self.handle.expect( "onos>" )
@@ -289,7 +294,10 @@
if re.search( "Error", response ):
return main.FALSE
return main.TRUE
-
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.cleanup()
+ main.exit()
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -3167,7 +3175,6 @@
else:
main.log.error( self.name + ": setTestAdd did not" +
" match expected output" )
- main.log.debug( self.name + " expected: " + pattern )
main.log.debug( self.name + " actual: " + repr( output ) )
return main.ERROR
except AssertionError:
@@ -3455,42 +3462,31 @@
main.cleanup()
main.exit()
- def counters( self ):
+ def counters( self, jsonFormat=True ):
"""
Command to list the various counters in the system.
returns:
- A dict containing the counters in the system with the counter
- names being the keys and the value of the counters being the
- values OR
+ if jsonFormat, a string of the json object returned by the cli
+ command
+ if not jsonFormat, the normal string output of the cli command
None on error
"""
- #FIXME: JSON FORMAT
try:
counters = {}
cmdStr = "counters"
+ if jsonFormat:
+ cmdStr += " -j"
output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
- for line in output.splitlines():
- match = re.search( "name=(\S+) value=(\d+)", line )
- if match:
- key = match.group( 1 )
- value = int( match.group( 2 ) )
- counters[key] = value
- else:
- main.log.error( self.name + ": counters did not match " +
- "expected output" )
- main.log.debug( self.name + " expected: " + pattern )
- main.log.debug( self.name + " actual: " + repr( output ) )
- return None
- return counters
+ return output
except AssertionError:
main.log.error( "Error in processing 'counters' command: " +
str( output ) )
- return main.ERROR
+ return None
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
- return main.ERROR
+ return None
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 09bc3d1..bb30335 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -98,7 +98,28 @@
response = main.FALSE
return response
- def onosPackage( self ):
+ def getEpochMs( self ):
+ """
+ Returns milliseconds since epoch
+
+ When checking multiple nodes in a for loop,
+ around a hundred milliseconds of difference (ascending) is
+ generally acceptable due to calltime of the function.
+ Few seconds, however, is not and it means clocks
+ are off sync.
+ """
+ try:
+ self.handle.sendline( 'date +%s.%N' )
+ self.handle.expect( 'date \+\%s\.\%N' )
+ self.handle.expect( '\$' )
+ epochMs = self.handle.before
+ return epochMs
+ except Exception:
+ main.log.exception( 'Uncaught exception getting epoch time' )
+ main.cleanup()
+ main.exit()
+
+ def onosPackage( self, opTimeout=30 ):
"""
Produce a self-contained tar.gz file that can be deployed
and executed on any platform with Java 7 JRE.
@@ -106,7 +127,7 @@
try:
self.handle.sendline( "onos-package" )
self.handle.expect( "onos-package" )
- self.handle.expect( "tar.gz", timeout=30 )
+ self.handle.expect( "tar.gz", opTimeout )
handle = str( self.handle.before )
main.log.info( "onos-package command returned: " +
handle )
@@ -153,7 +174,7 @@
main.cleanup()
main.exit()
- def cleanInstall( self ):
+ def cleanInstall( self, mciTimeout=600 ):
"""
Runs mvn clean install in the root of the ONOS directory.
This will clean all ONOS artifacts then compile each module
@@ -180,7 +201,7 @@
'BUILD\sSUCCESS',
'onos\$', #TODO: fix this to be more generic?
'ONOS\$',
- pexpect.TIMEOUT ], timeout=600 )
+ pexpect.TIMEOUT ], mciTimeout )
if i == 0:
main.log.error( self.name + ":There is insufficient memory \
for the Java Runtime Environment to continue." )
@@ -485,8 +506,10 @@
main.exit()
def getBranchName( self ):
+ main.log.info( "self.home = " )
+ main.log.info( self.home )
self.handle.sendline( "cd " + self.home )
- self.handle.expect( "ONOS\$" )
+ self.handle.expect( self.home + "\$" )
self.handle.sendline( "git name-rev --name-only HEAD" )
self.handle.expect( "git name-rev --name-only HEAD" )
self.handle.expect( "\$" )
@@ -1519,13 +1542,18 @@
except Exception:
main.log.exception( "Copying files failed" )
- def checkLogs( self, onosIp ):
+ def checkLogs( self, onosIp, restart=False):
"""
runs onos-check-logs on the given onos node
+ If restart is True, use the old version of onos-check-logs which
+ does not print the full stacktrace, but shows the entire log file,
+ including across restarts
returns the response
"""
try:
cmd = "onos-check-logs " + str( onosIp )
+ if restart:
+ cmd += " old"
self.handle.sendline( cmd )
self.handle.expect( cmd )
self.handle.expect( "\$" )
diff --git a/TestON/drivers/common/cli/quaggaclidriver.py b/TestON/drivers/common/cli/quaggaclidriver.py
index f391e3a..7e3df2b 100644
--- a/TestON/drivers/common/cli/quaggaclidriver.py
+++ b/TestON/drivers/common/cli/quaggaclidriver.py
@@ -219,8 +219,8 @@
intentsJsonObj = json.loads( getIntentsResult )
for intent in intentsJsonObj:
- if intent[ 'appId' ] != "org.onosproject.sdnip":
- continue
+ #if intent[ 'appId' ] != "org.onosproject.sdnip":
+ # continue
if intent[ 'type' ] == "MultiPointToSinglePointIntent" \
and intent[ 'state' ] == 'INSTALLED':
egress = str( intent[ 'egress' ][ 'device' ] ) + ":" \
@@ -242,6 +242,26 @@
intents.append( intent )
return sorted( intents )
+ # This method calculates the MultiPointToSinglePointIntent number installed
+ def extractActualRouteIntentNum( self, getIntentsResult ):
+ intentsJsonObj = json.loads( getIntentsResult )
+ num = 0
+ for intent in intentsJsonObj:
+ if intent[ 'type' ] == "MultiPointToSinglePointIntent" \
+ and intent[ 'state' ] == 'INSTALLED':
+ num = num + 1
+ return num
+
+ # This method calculates the PointToPointIntent number installed
+ def extractActualBgpIntentNum( self, getIntentsResult ):
+ intentsJsonObj = json.loads( getIntentsResult )
+ num = 0
+ for intent in intentsJsonObj:
+ if intent[ 'type' ] == "PointToPointIntent" \
+ and intent[ 'state' ] == 'INSTALLED':
+ num = num + 1
+ return num
+
# This method extracts all actual BGP intents from ONOS CLI
def extractActualBgpIntents( self, getIntentsResult ):
intents = []
@@ -250,8 +270,8 @@
intentsJsonObj = json.loads( getIntentsResult )
for intent in intentsJsonObj:
- if intent[ 'appId' ] != "org.onosproject.sdnip":
- continue
+ #if intent[ 'appId' ] != "org.onosproject.sdnip":
+ # continue
if intent[ 'type' ] == "PointToPointIntent" \
and "protocol=6" in str( intent[ 'selector' ] ):
ingress = str( intent[ 'ingress' ][ 'device' ] ) + ":" \
diff --git a/TestON/tests/FuncPlatform/Dependency/App.py b/TestON/tests/FuncPlatform/Dependency/App.py
new file mode 100644
index 0000000..5410c32
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/App.py
@@ -0,0 +1,68 @@
+"""
+Methods related to application interaction
+
+"""
+
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def activate( apps, nodeToActivateFrom=0 ):
+ """
+ Activate specified applications from node specified
+
+ Ex) apps = ['metrics', 'fwd']
+ nodeToActivateFrom = range( 0, nodes )
+ """
+ if isinstance( apps, ( int, basestring ) ):
+ main.log.error( 'Please pass in a list of strings for args' )
+ return main.FALSE
+
+ if not isinstance( nodeToActivateFrom, ( int ) ) or \
+ nodeToActivateFrom < 0:
+ main.log.error( 'Incorrect node specified' )
+ return main.FALSE
+
+ # TODO: Start log capture and listen for exceptions
+ # and errors. Also investigate possible keywords
+ # to listen for when activating applications
+
+
+ for app in apps:
+ # Check if app str in appList is in the main scope
+ # definition main.appList
+ if app not in main.appList:
+ main.log.error( 'Invalid app name given' )
+ return main.FALSE
+
+ try:
+ # NOTE: assumes node 1 is always activating application
+ appOutput = main.CLIs[nodeToActivateFrom].activateApp(
+ main.appList[app] )
+ except KeyError:
+ main.log.error( 'There was an error with the key '+
+ str(app) + '. Check the appList dictionary' )
+ return main.FALSE
+ except Exception:
+ main.log.error( 'Uncaught exception error while ' +
+ 'activating applications: ' + str(app) )
+ return main.FALSE
+
+ return main.TRUE
+
+def deactivate( apps, nodeToDeactivateFrom=0 ):
+ """
+ Deactivate specified applications from node specified
+
+ """
+ main.log.report( 'deactivate implment me' )
+
+def isAppInstallSuccess():
+ """
+ Check the app list across clusters to determine
+ that apps have been installed successfully
+
+ """
+
+ main.log.report( 'isAppInstallSuccess implement me' )
+
diff --git a/TestON/tests/FuncPlatform/Dependency/Logger.py b/TestON/tests/FuncPlatform/Dependency/Logger.py
new file mode 100644
index 0000000..22d3f61
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Logger.py
@@ -0,0 +1,57 @@
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def checkOnosLog( nodeIp, option='',
+ outputType=0):
+ """
+ Listens to the log for any Errors and Exceptions.
+
+ Runs 'onos-check-logs <option>'
+ This script only returns if there are any errors
+ or exceptions
+
+ outputType
+ 0: Return output of log
+ 1: Return (#Errors, #Exceptions, #Warn)
+
+ """
+ if not isinstance( option, basestring ):
+ main.log.error( 'Incorrect grep format specified' )
+ return main.FALSE
+
+ try:
+ main.log.info( 'Starting Onos-log listening for '+
+ str(option) )
+ cmd = 'onos-check-logs ' + str(nodeIp) + ' old'
+ if outputType == 0:
+ main.ONOSbench.handle.sendline( cmd )
+ main.ONOSbench.handle.expect( cmd )
+ main.ONOSbench.handle.expect('\$')
+ logResult = main.ONOSbench.handle.before
+ return logResult
+ elif outputType == 1:
+ # Important in assertion criteria
+ # to determine how much warn / error is
+ # acceptable
+ return 'Implement option 1'
+ else:
+ main.log.error( 'Incorrect outputType specified' )
+ return main.FALSE
+
+ except Exception:
+ main.log.exception( self.name + ': Uncaught exception' )
+ main.cleanup()
+ main.exit()
+
+def setLogLevel( level ):
+ """
+ Set the log level of onos
+ """
+ main.log.info( 'setLogLevel implement me' )
+
+def getLogReport( nodeIp, searchTerms ):
+ """
+ Refer to CLI driver for 'logReport'
+ """
+ main.log.info( 'getLogReport - implement me!' )
diff --git a/TestON/tests/FuncPlatform/Dependency/Shutdown.py b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
new file mode 100644
index 0000000..ba77bbd
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Shutdown.py
@@ -0,0 +1,19 @@
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def killOnosNodes( nodeIps ):
+ """
+ Kill all components of Onos on
+ given list of ips
+
+ Ex) nodeIps = ['10.0.0.1', '10.0.0.2']
+ """
+ killResult = main.TRUE
+
+ for node in nodeIps:
+ killResult = killResult and main.ONOSbench.onosDie( node )
+ if killResult == main.TRUE:
+ main.log.info( str(node) + ' was killed' )
+
+ return killResult
diff --git a/TestON/tests/FuncPlatform/Dependency/Startup.py b/TestON/tests/FuncPlatform/Dependency/Startup.py
new file mode 100644
index 0000000..f3f90a9
--- /dev/null
+++ b/TestON/tests/FuncPlatform/Dependency/Startup.py
@@ -0,0 +1,219 @@
+
+"""
+Startup related methods for ONOS
+
+Guidelines:
+ * Group sequential functionalities together
+ * Methods should not be dependent on platform
+ * Return main.TRUE on success or comprehensive error message
+ on failure (TBD)
+ * All methods should be consistent in expected behavior
+"""
+import time
+import json
+
+def __init__( self ):
+ self.ip = '127.0.0.1'
+
+def gitPullAndMci( branchName, commitLog=False ):
+ """
+ Pull from branch repository specified and compile changes
+ If commitLog is True, report commit information
+
+ Any errors / warnings will be handled by respective
+ driver function calls
+ """
+ co = main.ONOSbench.gitCheckout( branchName )
+ gp = main.ONOSbench.gitPull()
+ ci = main.ONOSbench.cleanInstall()
+
+ if co and gp and ci == main.TRUE:
+ if commitLog:
+ main.log.report( 'Commit information - ' )
+ main.ONOSbench.getVersion(report=True)
+
+ return main.TRUE
+
+ else:
+ # TODO: Comprehensive error message
+ return 'git pull and mci failed'
+
+def initOnosStartupSequence( cellName, appStr, benchIp, mnIp, onosIps ):
+ """
+ Startup sequence includes the following:
+ * Create cell file
+ * Set cell variables on ONOS bench
+ * Verify cell
+ * Create ONOS package
+ * Force install ONOS package
+ * Start ONOS service
+ * Start ONOS cli
+
+ Also verifies that Onos is up and running by
+ 'isup' driver function which executs
+ 'onos-wait-for-start'
+ """
+
+ # NOTE: leave out create cell file until bug addressed
+ #cf = main.ONOSbench.createCellFile( benchIp, cellName, mnIp,
+ # str(appStr), *onosIps )
+ numNodes = len(onosIps)
+
+ sc = main.ONOSbench.setCell( cellName )
+ vc = main.ONOSbench.verifyCell()
+ op = main.ONOSbench.onosPackage()
+ for addr in onosIps:
+ oi = main.ONOSbench.onosInstall( node = addr )
+
+ time.sleep( 5 )
+
+ iu = main.TRUE
+ for node in onosIps:
+ iu = iu and main.ONOSbench.isup( node )
+
+ cli = main.TRUE
+ for node in range( 0, numNodes ):
+ cli = cli and main.CLIs[node].startOnosCli( onosIps[node] )
+
+ # Check if all nodes are discovered correctly using
+ # 'nodes' command in Onos Cli
+ na = main.TRUE
+ try:
+ nodeCmdJson = json.loads( main.CLIs[0].nodes() )
+ for node in nodeCmdJson:
+ if node['state'] != 'ACTIVE':
+ main.log.warn( str( node['id'] ) +
+ ' Node is not in ACTIVE state.' )
+ na = main.FALSE
+ if na != main.FALSE:
+ main.log.info( 'All nodes discovered successfully' )
+ except Exception:
+ main.log.error( 'nodes command did not execute properly' )
+ return main.FALSE
+
+ if sc and vc and op and oi and iu and cli and na == main.TRUE:
+ return main.TRUE
+ else:
+ return main.FALSE
+
+def installOnosFromTar( wgetAddr, nodeIps ):
+ """
+ Install Onos directly from tar.gz file.
+ Due to the nature of the specific steps required
+ to startup Onos in this fashion, all commands
+ required to start Onos from tar.gz will be
+ grouped in this method.
+
+ 1) wget latest onos tar.gz on onos node
+ 2) untar package
+ 3) specify onos-config cluster
+ 4) start onos via onos-service
+ 5) form onos cluster using onos-form-cluster
+ 6) check for successful startup
+
+ Specify the download link for the tar.gz.
+ Provide a list of nodeIps
+
+ Ex) wgetAddr = 'https://mytargzdownload.com/file.tar.gz'
+ nodeIps = ['10.0.0.1', '10.0.0.2']
+ """
+ if isinstance( nodeIps, ( int, basestring ) ):
+ main.log.error( 'Please pass in a list of string nodes' )
+ return main.FALSE
+
+ # Obtain filename from provided address
+ # assumes that filename is separated by '/' character
+ f_name = ''
+ addr = str( wgetAddr ).split('/')
+ for phrase in addr:
+ if 'tar.gz' in phrase:
+ f_name = str( phrase )
+ main.log.info( 'Using ' + f_name + ' as file' )
+
+ clusterCount = len( nodeIps )
+
+ main.log.info( 'Initiating Onos installation sequence ' +
+ 'using tar.gz ... This may take a few minutes' )
+
+ for node in range( 0, clusterCount ):
+ # Use the wgetAddr to download a new tar.gz from server
+ try:
+ main.ONOSnode[node].handle.sendline( 'wget ' + wgetAddr )
+ main.ONOSnode[node].handle.expect( 'saved' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ main.log.info( 'Successfully downloaded tar.gz ' +
+ 'on node: ' + str( main.ONOSips[node] ) )
+ except Exception:
+ # NOTE: Additional exception may be appropriate
+ main.log.error( 'Uncaught exception while ' +
+ 'downloading Onos tar.gz: ' +
+ main.ONOSnode[node].handle.before )
+ return main.FALSE
+
+ for node in range( 0, clusterCount ):
+ # Untar files on all nodes, then enter the
+ # newly created directory
+ try:
+ main.ONOSnode[node].handle.sendline( 'tar zxvf ' + f_name )
+ # Verbose output of tar will contain some onos returns
+ main.ONOSnode[node].handle.expect( 'onos' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ # NOTE: Making an assumption here:
+ # the directory created by tar file has a name that
+ # starts with 'onos-' followed by version number
+ # '1.2.0'. If this is NOT true, this method must
+ # be changed to enter the correct directory.
+ # The directory name is currently dynamic
+ # and depends on the day at which you downloaded
+ # the tar file
+ # Enter onos- wildcard to disregard its version / date
+ # suffix
+ main.ONOSnode[node].handle.sendline( 'cd onos-*' )
+ main.ONOSnode[node].handle.expect( '\$' )
+
+ main.ONOSnode[node].handle.sendline( 'pwd' )
+ main.ONOSnode[node].handle.expect( 'pwd' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ pwd = main.ONOSnode[node].handle.before
+ if 'onos' in str(pwd):
+ main.log.info( 'tar zxvf ' + f_name + ' successful ' +
+ 'on node ' + str( main.ONOSips[node] ) )
+
+ except Exception:
+ main.log.error( 'Uncaught exception while executing ' +
+ 'tar zxvf of ' + f_name + ': ' +
+ main.ONOSnode[node].handle.before +
+ main.ONOSnode[node].handle.after)
+ return main.FALSE
+
+ for node in range( 0, clusterCount ):
+ try:
+ main.ONOSnode[node].handle.sendline( 'bin/onos-service '+
+ 'server &' )
+ # Send some extra characters to run the process
+ main.ONOSnode[node].handle.sendline( '' )
+ main.ONOSnode[node].handle.sendline( '' )
+ main.ONOSnode[node].handle.expect( '\$' )
+ except Exception:
+ main.log.error( 'Uncaught exception while executing ' +
+ 'onos-service server command ' +
+ str( main.ONOSnode[node].handle.before ) )
+ return main.FALSE
+
+ iu = main.TRUE
+ for node in nodeIps:
+ iu = iu and main.ONOSbench.isup( node )
+
+ if iu == main.TRUE:
+ return main.TRUE
+ else:
+ return main.FALSE
+
+def addAndStartOnosNode( nodeIps ):
+ """
+ A scale-out scenario that adds specified list of
+ nodes and starts those instances.
+
+ Ex) nodeIps = ['10.0.0.2', '10.0.0.3', 10.0.0.4']
+ """
+ main.log.info( 'addAndStartOnosNode implement me!' )
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.params b/TestON/tests/FuncPlatform/FuncPlatform.params
new file mode 100644
index 0000000..3895660
--- /dev/null
+++ b/TestON/tests/FuncPlatform/FuncPlatform.params
@@ -0,0 +1,63 @@
+<PARAMS>
+ <testcases>1,2,3,4</testcases>
+ <DEP>
+ <startupSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Startup.py
+ </startupSrc>
+ <startupClassName>
+ Startup
+ </startupClassName>
+
+ <appSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/App.py
+ </appSrc>
+ <appClassName>
+ App
+ </appClassName>
+
+ <logSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Logger.py
+ </logSrc>
+ <logClassName>
+ Log
+ </logClassName>
+
+ <shutdownSrc>
+ /home/admin/ONLabTest/TestON/tests/FuncPlatform/Dependency/Shutdown.py
+ </shutdownSrc>
+ <shutdownClassName>
+ Shutdown
+ </shutdownClassName>
+
+ <targz>
+ http://downloads.onosproject.org/nightly/onos-1.2.0.latest-NIGHTLY.tar.gz
+ </targz>
+ </DEP>
+
+ <CTRL>
+ <num>3</num>
+ <port>6633</port>
+ <ip1>10.128.174.1</ip1>
+ <ip2>10.128.174.2</ip2>
+ <ip3>10.128.174.3</ip3>
+ </CTRL>
+
+ <MN>
+ <ip>10.128.10.90</ip>
+ </MN>
+
+ <BENCH>
+ <ip>10.128.174.10</ip>
+ </BENCH>
+
+ <CELL>
+ <name>func_plat</name>
+ <appStr>"drivers,openflow"</appStr>
+ </CELL>
+
+ <GIT>
+ <branchName>master</branchName>
+ <pull>off</pull>
+ </GIT>
+
+</PARAMS>
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.py b/TestON/tests/FuncPlatform/FuncPlatform.py
new file mode 100644
index 0000000..60b1aa0
--- /dev/null
+++ b/TestON/tests/FuncPlatform/FuncPlatform.py
@@ -0,0 +1,194 @@
+"""
+FuncPlatform
+
+A functional test designed to test the environment and
+gather information on startup -> shutdown related issues.
+
+Future works may also include security mode startup /
+shutdown check and cfg get and set.
+
+Abstracting the collection of commands that go hand in hand
+should allow easy rearrangement of steps to replicate or
+create scenarios.
+For example:
+ CASE N - Represents a particular scenario
+ Steps - Represents abstraction methods called from
+ dependency
+ 1. Bring ONOS 1 up
+ 2. Activate application X
+ 3. Activate application Y
+ 4. Deactivate application X
+
+The ideal platform test script should have incredible
+robustness to possible exceptions and report the most
+useful error messages.
+
+contributers to contact for help:
+andrew@onlab.us
+"""
+
+class FuncPlatform:
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ """
+ Main scope initialization case
+ Must include to run any other test cases
+ """
+ import imp
+
+ # NOTE: Hardcoded application name subject to change
+ # closely monitor and make changes when necessary
+ # (or implement ways to dynamically get names)
+ main.appList = {
+ 'bgprouter' : 'org.onosproject.bgprouter',
+ 'config' : 'org.onosproject.config',
+ 'cordfabric' : 'org.onosproject.cordfabric',
+ 'demo' : 'org.onosproject.demo',
+ 'distributedprimitives' : 'org.onosproject.distributedprimitives',
+ 'election' : 'org.onosproject.election',
+ 'flowrule' : 'org.onosproject.flowrule',
+ 'fwd' : 'org.onosproject.fwd',
+ 'intentperf' : 'org.onosproject.intentperf',
+ 'messagingperf' : 'org.onosproject.messagingperf',
+ 'metrics' : 'org.onosproject.metrics',
+ 'mobility' : 'org.onosproject.mobility',
+ 'netconf' : 'org.onosproject.netconf',
+ 'null' : 'org.onosproject.null',
+ 'optical' : 'org.onosproject.optical',
+ 'pcep' : 'org.onosproject.pcep',
+ 'proxyarp' : 'org.onosproject.proxyarp',
+ 'reactive.routing' : 'org.onosproject.reactive.routing',
+ 'sdnip' : 'org.onosproject.sdnip',
+ 'segmentrouting' : 'org.onosproject.segmentrouting',
+ 'tunnel' : 'org.onosproject.tunnel',
+ 'virtualbng' : 'org.onosproject.virtualbng',
+ 'xosintegration' : 'org.onosproject.xosintegration'
+ }
+ # List of ONOS ip's specififed in params
+ main.ONOSips = []
+ main.CLIs = []
+ main.ONOSnode = []
+
+ for node in range( 0, int(main.params['CTRL']['num']) ):
+ main.ONOSips.append( main.params['CTRL']['ip'+str(node+1)] )
+ main.CLIs.append(
+ getattr( main, 'ONOS' + str(node+1) + 'cli' ) )
+ main.ONOSnode.append(
+ getattr( main, 'ONOS' + str(node+1) ) )
+
+ # Application source and name definitions
+ startupSrc = main.params['DEP']['startupSrc']
+ startupClassName = main.params['DEP']['startupClassName']
+
+ appClassName = main.params['DEP']['appClassName']
+ appSrc = main.params['DEP']['appSrc']
+
+ logClassName = main.params['DEP']['logClassName']
+ logSrc = main.params['DEP']['logSrc']
+
+ shutdownClassName = main.params['DEP']['shutdownClassName']
+ shutdownSrc = main.params['DEP']['shutdownSrc']
+
+ # Importing dependency class(es)
+ # Refer to source files in Dependency folder to
+ # make changes to its respective methods
+ # Be weary of naming collisions
+ try:
+ main.startup = imp.load_source( startupClassName, startupSrc )
+ main.app = imp.load_source( appClassName, appSrc )
+ main.onosLog = imp.load_source( logClassName, logSrc )
+ main.shutdown = imp.load_source( shutdownClassName, shutdownSrc )
+ except ImportError:
+ main.log.error( 'Error importing class file(s). Please ' +
+ 'check file location' )
+ main.cleanup()
+ main.exit()
+
+ def CASE2( self, main ):
+ import time
+
+ cellName = main.params['CELL']['name']
+ appStr = main.params['CELL']['appStr']
+ benchIp = main.params['BENCH']['ip']
+ branchName = main.params['GIT']['branchName']
+ gitPull = main.params['GIT']['pull']
+ mnIp = main.params['MN']['ip']
+
+ main.case( 'Setup environment and install ONOS' )
+ if gitPull == 'on':
+ main.step( 'Git pull and clean install' )
+ gitPullResult = main.startup.gitPullAndMci( branchName )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=gitPullResult,
+ onpass='Git pull and install successful',
+ onfail='Git pull and install failed: ' +
+ str(gitPullResult) )
+
+ main.step( 'Initiate ONOS startup sequence' )
+ startupResult = main.startup.initOnosStartupSequence(
+ cellName, appStr, benchIp, mnIp, main.ONOSips )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=startupResult,
+ onpass='ONOS startup sequence successful',
+ onfail='ONOS startup sequence failed: ' +
+ str(startupResult) )
+
+ def CASE3( self, main ):
+ import time
+
+ main.case( 'Activate applications and check installation' )
+
+ # NOTE: Test only
+ # Unceremoniously kill onos 2
+ main.ONOSbench.onosDie( '10.128.174.2' )
+
+ time.sleep( 30 )
+
+ main.step( 'Sample Onos log check' )
+ logResult = main.onosLog.checkOnosLog( main.ONOSips[0] )
+ main.log.info( logResult )
+ # TODO: Define pass criteria
+ utilities.assert_equals( expect=main.TRUE,
+ actual=main.TRUE,
+ onpass= 'Logging successful',
+ onfail= 'Logging failed ' )
+
+ # Sample app activation
+ main.step( 'Activating applications metrics and fwd' )
+ appList = ['metrics', 'fwd']
+ appResult = main.app.activate( appList )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=appResult,
+ onpass= 'App activation of ' + str(appList) + ' successful',
+ onfail= 'App activation failed ' + str(appResult) )
+
+ def CASE4( self, main ):
+ """
+ Download ONOS tar.gz built from latest nightly
+ (following tutorial on wiki) and run ONOS directly on the
+ instance
+ """
+ import imp
+
+ targz = main.params['DEP']['targz']
+ clusterCount = main.params['CTRL']['num']
+
+ main.case( 'Install ONOS from onos.tar.gz file' )
+
+ main.step( 'Killing all ONOS instances previous started' )
+ killResult = main.shutdown.killOnosNodes( main.ONOSips )
+ utilities.assert_equals( expect=main.TRUE,
+ actual = killResult,
+ onpass = 'All Onos nodes successfully killed',
+ onfail = 'Onos nodes were not successfully killed' )
+
+ main.step( 'Starting ONOS using tar.gz on all nodes' )
+ installResult = main.startup.installOnosFromTar( targz, main.ONOSips )
+ utilities.assert_equals( expect=main.TRUE,
+ actual = installResult,
+ onpass= 'Onos tar.gz installation successful',
+ onfail= 'Onos tar.gz installation failed' )
+
+
diff --git a/TestON/tests/FuncPlatform/FuncPlatform.topo b/TestON/tests/FuncPlatform/FuncPlatform.topo
new file mode 100644
index 0000000..f19e8f8
--- /dev/null
+++ b/TestON/tests/FuncPlatform/FuncPlatform.topo
@@ -0,0 +1,82 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>10.128.174.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOSbench>
+
+ <ONOS1>
+ <host>10.128.174.1</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS1>
+
+ <ONOS2>
+ <host>10.128.174.2</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS2>
+
+ <ONOS3>
+ <host>10.128.174.3</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS3>
+
+ <ONOS1cli>
+ <host>10.128.174.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS1cli>
+
+ <ONOS2cli>
+ <host>10.128.174.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS2cli>
+
+ <ONOS3cli>
+ <host>10.128.174.10</host>
+ <user>admin</user>
+ <password></password>
+ <type>OnosCliDriver</type>
+ <connect_order>8</connect_order>
+ <COMPONENTS> </COMPONENTS>
+ </ONOS3cli>
+
+ <MininetOvs>
+ <host>10.128.10.90</host>
+ <user>admin</user>
+ <password></password>
+ <type>MininetCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ <arg1> --custom topo-perf-2sw.py </arg1>
+ <arg2> --arp --mac --topo mytopo </arg2>
+ <arg3> </arg3>
+ <controller> remote </controller>
+ </COMPONENTS>
+ </MininetOvs>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
index 500c3ad..390facf 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -3,9 +3,10 @@
<ENV>
<cellName>HA</cellName>
</ENV>
- <Git>False</Git>
+ <Git> False </Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index bb69fcc..50bf06f 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -46,7 +46,7 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
+ main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
main.caseExplaination = "Setup the test environment including " +\
@@ -95,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -126,15 +126,15 @@
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean install" )
cleanInstallResult = main.TRUE
- if gitPullResult == main.TRUE:
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
utilities.assert_equals( expect=main.TRUE,
actual=cleanInstallResult,
onpass="MCI successful",
@@ -145,10 +145,12 @@
# Plot Name = Plot-HA, only can be used if multiple plots
# index = The number of the graph under plot name
job = "HAClusterRestart"
+ plotName = "Plot-HA"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
- '/plot/getPlot?index=0&width=500&height=300"' +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
'noborder="0" width="500" height="300" scrolling="yes" ' +\
'seamless="seamless"></iframe>\n'
graphs += ']]></ac:plain-text-body>\n'
@@ -177,7 +179,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
@@ -204,12 +206,13 @@
onpass="ONOS cli startup successful",
onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
appCheck = main.TRUE
@@ -292,8 +295,6 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -538,7 +539,6 @@
pass # intent submitted is in onos
else:
intentAddResult = False
- # FIXME: DEBUG
if intentAddResult:
intentStop = time.time()
else:
@@ -639,7 +639,15 @@
main.log.debug( "Intents in " + cli.name + ": " +
str( sorted( onosIds ) ) )
if sorted( ids ) != sorted( intentIds ):
+ # IF intents are missing
correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ correct = False
+ break
if correct:
break
else:
@@ -777,7 +785,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -789,9 +797,6 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
@@ -799,31 +804,37 @@
onfail="Intents have not been installed correctly, pings failed." )
main.step( "Check Intent state" )
- installedCheck = True
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ installedCheck = False
count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- # Print states
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
+ while not installedCheck and count < 40:
+ installedCheck = True
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
utilities.assert_equals( expect=True, actual=installedCheck,
onpass="Intents are all INSTALLED",
- onfail="Intents are not all in " +\
+ onfail="Intents are not all in " +
"INSTALLED state" )
main.step( "Check leadership of topics" )
@@ -894,11 +905,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
-
+ main.log.info( "Waiting 60 seconds to see if the state of " +
+ "intents change" )
+ time.sleep( 60 )
if not installedCheck:
- main.log.info( "Waiting 60 seconds to see if the state of " +
- "intents change" )
- time.sleep( 60 )
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
@@ -974,9 +984,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- main.log.debug( CLIs[0].flows( jsonFormat=False ) )
-
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
+ # the wait is above
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -989,7 +1000,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -1068,7 +1079,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1082,7 +1093,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1130,7 +1141,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1143,11 +1154,11 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1231,7 +1242,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1255,7 +1266,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1383,6 +1394,14 @@
for t in threads:
t.join()
ports.append( t.result )
+ try:
+ # FIXME: DEBUG
+ main.log.debug( json.dumps( json.loads( t.result ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except:
+ main.log.debug( repr( t.result ) )
links = []
threads = []
for i in range( numControllers ):
@@ -1418,14 +1437,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1461,12 +1480,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1568,6 +1587,15 @@
iCounterValue = 0
main.case( "Restart entire ONOS cluster" )
+ # FIXME: DEBUG
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ # FIXME: DEBUG
+
main.step( "Killing ONOS nodes" )
killResults = main.TRUE
killTime = time.time()
@@ -1584,7 +1612,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
@@ -1608,25 +1636,25 @@
utilities.assert_equals( expect=main.TRUE, actual=cliResults,
onpass="ONOS cli started",
onfail="ONOS clis did not restart" )
-
+
# Grab the time of restart so we chan check how long the gossip
# protocol has had time to work
main.restartTime = time.time() - killTime
main.log.debug( "Restart time: " + str( main.restartTime ) )
- '''
+
# FIXME: revisit test plan for election with madan
# Rerun for election on restarted nodes
- run1 = CLIs[0].electionTestRun()
- run2 = CLIs[1].electionTestRun()
- run3 = CLIs[2].electionTestRun()
- ...
- ...
- runResults = run1 and run2 and run3
+ runResults = main.TRUE
+ for cli in CLIs:
+ run = CLIs[0].electionTestRun()
+ if run != main.TRUE:
+ main.log.error( "Error running for election on " + cli.name )
+ runResults = runResults and run
utilities.assert_equals( expect=main.TRUE, actual=runResults,
onpass="Reran for election",
onfail="Failed to rerun for election" )
- '''
- # TODO: Make this configurable
+
+ # TODO: Make this configurable
time.sleep( 60 )
main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
@@ -1683,7 +1711,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1697,7 +1725,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1773,7 +1801,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1786,7 +1814,7 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
@@ -1854,22 +1882,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1898,8 +1945,8 @@
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables" )
- main.step( "Check the continuous pings to ensure that no packets " +
- "were dropped during component failure" )
+ # main.step( "Check the continuous pings to ensure that no packets " +
+ # "were dropped during component failure" )
main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
main.params[ 'TESTONIP' ] )
LossInPings = main.FALSE
@@ -1917,12 +1964,13 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
- utilities.assert_equals(
- expect=main.FALSE,
- actual=LossInPings,
- onpass="No Loss of connectivity",
- onfail="Loss of dataplane connectivity detected" )
+ main.log.info( "No loss of dataplane connectivity" )
+ # utilities.assert_equals(
+ # expect=main.FALSE,
+ # actual=LossInPings,
+ # onpass="No Loss of connectivity",
+ # onfail="Loss of dataplane connectivity detected" )
+
# NOTE: Since intents are not persisted with IntnentStore,
# we expect loss in dataplane connectivity
LossInPings = main.FALSE
@@ -1936,14 +1984,13 @@
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.report( cli.name +
- " shows no leader for the election-app was" +
- " elected after the old one died" )
+ main.log.error( cli.name +
+ " shows no leader for the election-app." )
leaderResult = main.FALSE
if len( set( leaderList ) ) != 1:
leaderResult = main.FALSE
@@ -2003,11 +2050,12 @@
main.step( "Collecting topology information from ONOS" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and elapsed < 120:
count += 1
if count > 1:
# TODO: Deprecate STS usage
MNTopo = TestONTopology( main.Mininet1, ctrls )
+ time.sleep( 1 )
cliStart = time.time()
devices = []
threads = []
@@ -2058,6 +2106,14 @@
for t in threads:
t.join()
ports.append( t.result )
+ try:
+ # FIXME: DEBUG
+ main.log.debug( json.dumps( json.loads( t.result ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except:
+ main.log.debug( repr( t.result ) )
links = []
threads = []
for i in range( numControllers ):
@@ -2085,6 +2141,7 @@
elapsed = time.time() - startTime
cliTime = time.time() - cliStart
+ print "Elapsed time: " + str( elapsed )
print "CLI time: " + str( cliTime )
for controller in range( numControllers ):
@@ -2157,13 +2214,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2184,13 +2241,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2291,7 +2348,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2320,7 +2376,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2348,14 +2403,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2394,11 +2448,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2447,9 +2499,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2505,7 +2555,7 @@
for node in nodes:
print colors[ 'purple' ] + "Checking logs for errors on " + \
node.name + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( node.ip_address )
+ print main.ONOSbench.checkLogs( node.ip_address, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
@@ -2513,8 +2563,9 @@
try:
timerLog = open( main.logdir + "/Timers.csv", 'w')
# Overwrite with empty line and close
- timerLog.write( "Restart\n" )
- timerLog.write( str( main.restartTime ) )
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
timerLog.close()
except NameError, e:
main.log.exception(e)
@@ -2546,7 +2597,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2582,7 +2633,6 @@
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
# NOTE: Need to re-run since being a canidate is not persistant
main.step( "Run for election on each node" )
@@ -2593,7 +2643,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2647,14 +2697,14 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
@@ -2671,10 +2721,10 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
utilities.assert_equals(
@@ -2714,6 +2764,7 @@
"""
Install Distributed Primitives app
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -2743,7 +2794,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
- time.sleep (5 ) # To allow all nodes to activate
+ time.sleep( 5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2803,7 +2854,7 @@
# Check that counter incremented numController times
pCounterResults = True
for i in addedPValues:
- tmpResult = i in pCounters
+ tmpResult = i in pCounters
pCounterResults = pCounterResults and tmpResult
if not tmpResult:
main.log.error( str( i ) + " is not in partitioned "
@@ -2873,28 +2924,41 @@
main.step( "Counters we added have the correct values" )
correctResults = main.TRUE
for i in range( numControllers ):
- current = onosCounters[i]
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
try:
- pValue = current.get( pCounterName )
- iValue = current.get( iCounterName )
- if pValue == pCounterValue:
- main.log.info( "Partitioned counter value is correct" )
- else:
- main.log.error( "Partitioned counter value is incorrect," +
- " expected value: " + str( pCounterValue )
- + " current value: " + str( pValue ) )
- correctResults = main.FALSE
- if iValue == iCounterValue:
- main.log.info( "In memory counter value is correct" )
- else:
- main.log.error( "In memory counter value is incorrect, " +
- "expected value: " + str( iCounterValue ) +
- " current value: " + str( iValue ) )
- correctResults = main.FALSE
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
except AttributeError, e:
main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
"is not as expected" )
correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=correctResults,
onpass="Added counters are correct",
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
index ca1f915..956d889 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -6,6 +6,7 @@
<Git>False</Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 256561a..af96bb5 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -46,7 +46,7 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS HA test: Restart minority of ONOS nodes - " +
+ main.log.info( "ONOS HA test: Restart minority of ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
main.caseExplaination = "Setup the test environment including " +\
@@ -95,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -126,15 +126,15 @@
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean install" )
cleanInstallResult = main.TRUE
- if gitPullResult == main.TRUE:
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
utilities.assert_equals( expect=main.TRUE,
actual=cleanInstallResult,
onpass="MCI successful",
@@ -145,10 +145,12 @@
# Plot Name = Plot-HA, only can be used if multiple plots
# index = The number of the graph under plot name
job = "HAMinorityRestart"
+ plotName = "Plot-HA"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
- '/plot/getPlot?index=0&width=500&height=300"' +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
'noborder="0" width="500" height="300" scrolling="yes" ' +\
'seamless="seamless"></iframe>\n'
graphs += ']]></ac:plain-text-body>\n'
@@ -177,7 +179,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
@@ -204,12 +206,13 @@
onpass="ONOS cli startup successful",
onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
appCheck = main.TRUE
@@ -636,6 +639,13 @@
str( sorted( onosIds ) ) )
if sorted( ids ) != sorted( intentIds ):
correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ correct = False
+ break
if correct:
break
else:
@@ -773,7 +783,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -785,9 +795,6 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
@@ -795,31 +802,37 @@
onfail="Intents have not been installed correctly, pings failed." )
main.step( "Check Intent state" )
- installedCheck = True
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ installedCheck = False
count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- # Print states
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
+ while not installedCheck and count < 40:
+ installedCheck = True
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
utilities.assert_equals( expect=True, actual=installedCheck,
onpass="Intents are all INSTALLED",
- onfail="Intents are not all in " +\
+ onfail="Intents are not all in " +
"INSTALLED state" )
main.step( "Check leadership of topics" )
@@ -970,9 +983,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- main.log.debug( CLIs[0].flows( jsonFormat=False ) )
-
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
+ # the wait is above
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -985,7 +999,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -1064,7 +1078,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1078,7 +1092,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1126,7 +1140,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1139,11 +1153,11 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1227,7 +1241,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1251,7 +1265,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1414,14 +1428,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1457,12 +1471,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1614,10 +1628,6 @@
main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
- time.sleep( 200 )
- main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
- main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
- main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
def CASE7( self, main ):
"""
@@ -1669,7 +1679,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1683,7 +1693,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1757,7 +1767,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1770,7 +1780,7 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
@@ -1837,22 +1847,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1900,7 +1929,7 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
+ main.log.info( "No loss of dataplane connectivity" )
utilities.assert_equals(
expect=main.FALSE,
actual=LossInPings,
@@ -1920,17 +1949,17 @@
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.report( cli.name +
+ main.log.error( cli.name +
" shows no leader for the election-app was" +
" elected after the old one died" )
leaderResult = main.FALSE
elif leaderN in restarted:
- main.log.report( cli.name + " shows " + str( leaderN ) +
+ main.log.error( cli.name + " shows " + str( leaderN ) +
" as leader for the election-app, but it " +
"was restarted" )
leaderResult = main.FALSE
@@ -2032,7 +2061,7 @@
for host in hosts[ controller ]:
if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ips on controller" +
+ "DEBUG:Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -2146,13 +2175,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2173,13 +2202,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2280,7 +2309,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2309,7 +2337,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2337,14 +2364,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2383,11 +2409,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2436,9 +2460,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2494,7 +2516,7 @@
for node in nodes:
print colors[ 'purple' ] + "Checking logs for errors on " + \
node.name + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( node.ip_address )
+ print main.ONOSbench.checkLogs( node.ip_address, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
@@ -2502,8 +2524,9 @@
try:
timerLog = open( main.logdir + "/Timers.csv", 'w')
# Overwrite with empty line and close
- timerLog.write( "Restart\n" )
- timerLog.write( str( main.restartTime ) )
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
timerLog.close()
except NameError, e:
main.log.exception(e)
@@ -2535,7 +2558,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2571,7 +2594,6 @@
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
main.step( "Check that each node shows the same leader" )
@@ -2621,14 +2643,14 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
@@ -2645,10 +2667,10 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
utilities.assert_equals(
@@ -2688,6 +2710,7 @@
"""
Install Distributed Primitives app
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -2717,7 +2740,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
- time.sleep (5 ) # To allow all nodes to activate
+ time.sleep( 5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2777,7 +2800,7 @@
# Check that counter incremented numController times
pCounterResults = True
for i in addedPValues:
- tmpResult = i in pCounters
+ tmpResult = i in pCounters
pCounterResults = pCounterResults and tmpResult
if not tmpResult:
main.log.error( str( i ) + " is not in partitioned "
@@ -2847,28 +2870,41 @@
main.step( "Counters we added have the correct values" )
correctResults = main.TRUE
for i in range( numControllers ):
- current = onosCounters[i]
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
try:
- pValue = current.get( pCounterName )
- iValue = current.get( iCounterName )
- if pValue == pCounterValue:
- main.log.info( "Partitioned counter value is correct" )
- else:
- main.log.error( "Partitioned counter value is incorrect," +
- " expected value: " + str( pCounterValue )
- + " current value: " + str( pValue ) )
- correctResults = main.FALSE
- if iValue == iCounterValue:
- main.log.info( "In memory counter value is correct" )
- else:
- main.log.error( "In memory counter value is incorrect, " +
- "expected value: " + str( iCounterValue ) +
- " current value: " + str( iValue ) )
- correctResults = main.FALSE
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
except AttributeError, e:
main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
"is not as expected" )
correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=correctResults,
onpass="Added counters are correct",
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
index e1421cd..f82aba3 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
@@ -25,6 +25,7 @@
<Git> False </Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
index e4658e7..d2a2ec1 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
@@ -217,12 +217,13 @@
cliResults = cliResult1 and cliResult2 and cliResult3 and\
cliResult4 and cliResult5 and cliResult6 and cliResult7
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
case1Result = ( cleanInstallResult and packageResult and
cellResult and verifyResult and onosInstallResult
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index 2440ec6..9106d68 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -25,6 +25,7 @@
<Git> True </Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index fe513ac..60c438e 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -47,7 +47,7 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS HA Sanity test - initialization" )
+ main.log.info( "ONOS HA Sanity test - initialization" )
main.case( "Setting up test environment" )
main.caseExplaination = "Setup the test environment including " +\
"installing ONOS, starting Mininet and ONOS" +\
@@ -95,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -126,15 +126,15 @@
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean install" )
cleanInstallResult = main.TRUE
- if gitPullResult == main.TRUE:
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
utilities.assert_equals( expect=main.TRUE,
actual=cleanInstallResult,
onpass="MCI successful",
@@ -179,7 +179,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
@@ -206,12 +206,13 @@
onpass="ONOS cli startup successful",
onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
appCheck = main.TRUE
@@ -294,13 +295,12 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
+ # FIXME: Check topo here
main.step( "Assign mastership of switches to specific controllers" )
# Manually assign mastership to the controller we want
@@ -634,7 +634,7 @@
main.log.error( "Error in pushing host intents to ONOS" )
main.step( "Intent Anti-Entropy dispersion" )
- for i in range(100):
+ for i in range(80):
correct = True
main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
for cli in CLIs:
@@ -644,11 +644,19 @@
main.log.debug( "Intents in " + cli.name + ": " +
str( sorted( onosIds ) ) )
if sorted( ids ) != sorted( intentIds ):
+ main.log.debug( "Set of intent IDs doesn't match" )
correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ main.log.warn( "Intent " + intent[ 'id' ] +
+ " is " + intent[ 'state' ] )
+ correct = False
+ break
if correct:
break
- else:
- time.sleep(1)
if not intentStop:
intentStop = time.time()
global gossipTime
@@ -782,7 +790,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -794,9 +802,6 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
@@ -804,31 +809,37 @@
onfail="Intents have not been installed correctly, pings failed." )
main.step( "Check Intent state" )
- installedCheck = True
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ installedCheck = False
count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- # Print states
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
+ while not installedCheck and count < 40:
+ installedCheck = True
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
utilities.assert_equals( expect=True, actual=installedCheck,
onpass="Intents are all INSTALLED",
- onfail="Intents are not all in " +\
+ onfail="Intents are not all in " +
"INSTALLED state" )
main.step( "Check leadership of topics" )
@@ -979,9 +990,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- main.log.debug( CLIs[0].flows( jsonFormat=False ) )
-
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
+ # the wait is above
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -994,7 +1006,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -1011,15 +1023,6 @@
actual=PingResult,
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
-
-
-
- '''
- #DEBUG
- if PingResult == main.FALSE:
- import time
- time.sleep( 100000 )
- '''
def CASE5( self, main ):
"""
@@ -1082,7 +1085,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1096,7 +1099,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1144,7 +1147,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1157,11 +1160,11 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1179,22 +1182,27 @@
for n in range( numControllers ):
title += " " * 10 + "ONOS" + str( n + 1 )
main.log.warn( title )
- # get all intent keys in the cluster
keys = []
- for nodeStr in ONOSIntents:
- node = json.loads( nodeStr )
- for intent in node:
- keys.append( intent.get( 'id' ) )
- keys = set( keys )
- for key in keys:
- row = "%-13s" % key
+ try:
+ # Get the set of all intent keys
for nodeStr in ONOSIntents:
node = json.loads( nodeStr )
for intent in node:
- if intent.get( 'id', "Error" ) == key:
- row += "%-15s" % intent.get( 'state' )
- main.log.warn( row )
- # End table view
+ keys.append( intent.get( 'id' ) )
+ keys = set( keys )
+ # For each intent key, print the state on each node
+ for key in keys:
+ row = "%-13s" % key
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ if intent.get( 'id', "Error" ) == key:
+ row += "%-15s" % intent.get( 'state' )
+ main.log.warn( row )
+ # End of intent state table
+ except ValueError as e:
+ main.log.exception( e )
+ main.log.debug( "nodeStr was: " + repr( nodeStr ) )
if intentsResults and not consistentIntents:
# print the json objects
@@ -1245,7 +1253,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1269,7 +1277,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1432,14 +1440,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1475,12 +1483,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1632,7 +1640,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1646,7 +1654,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1720,7 +1728,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1733,7 +1741,7 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
@@ -1800,22 +1808,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1863,7 +1890,7 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
+ main.log.info( "No loss of dataplane connectivity" )
utilities.assert_equals(
expect=main.FALSE,
actual=LossInPings,
@@ -1886,13 +1913,13 @@
pass
elif leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( cli.name + " sees " + str( leaderN ) +
+ main.log.error( cli.name + " sees " + str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
utilities.assert_equals(
@@ -1988,7 +2015,7 @@
for host in hosts[ controller ]:
if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ips on controller" +
+ "DEBUG:Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -2102,13 +2129,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2129,13 +2156,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2236,7 +2263,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2265,7 +2291,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2293,14 +2318,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2339,11 +2363,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2392,9 +2414,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2458,8 +2478,9 @@
try:
timerLog = open( main.logdir + "/Timers.csv", 'w')
# Overwrite with empty line and close
- timerLog.write( "Gossip Intents\n" )
- timerLog.write( str( gossipTime ) )
+ labels = "Gossip Intents"
+ data = str( gossipTime )
+ timerLog.write( labels + "\n" + data )
timerLog.close()
except NameError, e:
main.log.exception(e)
@@ -2491,7 +2512,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2527,7 +2548,6 @@
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
main.step( "Check that each node shows the same leader" )
@@ -2577,14 +2597,14 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
@@ -2601,10 +2621,10 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
utilities.assert_equals(
@@ -2644,6 +2664,7 @@
"""
Install Distributed Primitives app
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -2673,7 +2694,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
- time.sleep (5 ) # To allow all nodes to activate
+ time.sleep( 5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2733,7 +2754,7 @@
# Check that counter incremented numController times
pCounterResults = True
for i in addedPValues:
- tmpResult = i in pCounters
+ tmpResult = i in pCounters
pCounterResults = pCounterResults and tmpResult
if not tmpResult:
main.log.error( str( i ) + " is not in partitioned "
@@ -2803,28 +2824,41 @@
main.step( "Counters we added have the correct values" )
correctResults = main.TRUE
for i in range( numControllers ):
- current = onosCounters[i]
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
try:
- pValue = current.get( pCounterName )
- iValue = current.get( iCounterName )
- if pValue == pCounterValue:
- main.log.info( "Partitioned counter value is correct" )
- else:
- main.log.error( "Partitioned counter value is incorrect," +
- " expected value: " + str( pCounterValue )
- + " current value: " + str( pValue ) )
- correctResults = main.FALSE
- if iValue == iCounterValue:
- main.log.info( "In memory counter value is correct" )
- else:
- main.log.error( "In memory counter value is incorrect, " +
- "expected value: " + str( iCounterValue ) +
- " current value: " + str( iValue ) )
- correctResults = main.FALSE
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
except AttributeError, e:
main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
"is not as expected" )
correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=correctResults,
onpass="Added counters are correct",
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
index f057592..108ab88 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
@@ -23,6 +23,7 @@
<Git>False</Git>
<branch> master </branch>
<num_controllers> 1 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index 7194a3b..8f3c604 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -46,7 +46,7 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS Single node cluster restart " +
+ main.log.info( "ONOS Single node cluster restart " +
"HA test - initialization" )
main.case( "Setting up test environment" )
main.caseExplaination = "Setup the test environment including " +\
@@ -126,15 +126,15 @@
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean install" )
cleanInstallResult = main.TRUE
- if gitPullResult == main.TRUE:
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
utilities.assert_equals( expect=main.TRUE,
actual=cleanInstallResult,
onpass="MCI successful",
@@ -145,10 +145,12 @@
# Plot Name = Plot-HA, only can be used if multiple plots
# index = The number of the graph under plot name
job = "HASingleInstanceRestart"
+ plotName = "Plot-HA"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
- '/plot/getPlot?index=0&width=500&height=300"' +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
'noborder="0" width="500" height="300" scrolling="yes" ' +\
'seamless="seamless"></iframe>\n'
graphs += ']]></ac:plain-text-body>\n'
@@ -185,12 +187,13 @@
onpass="ONOS cli startup successful",
onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
appCheck = main.ONOScli1.appToIDCheck()
@@ -681,7 +684,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -693,9 +696,6 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.info(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
@@ -727,7 +727,7 @@
( str( count ), str( i ), str( s ) ) )
utilities.assert_equals( expect=True, actual=installedCheck,
onpass="Intents are all INSTALLED",
- onfail="Intents are not all in " +\
+ onfail="Intents are not all in " +
"INSTALLED state" )
main.step( "Check leadership of topics" )
@@ -878,9 +878,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- main.log.debug( main.ONOScli1.flows( jsonFormat=False ) )
-
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
+ # the wait is above
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -893,7 +894,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -943,7 +944,7 @@
ONOS1Mastership = main.ONOScli1.roles()
# TODO: Make this a meaningful check
if "Error" in ONOS1Mastership or not ONOS1Mastership:
- main.log.report( "Error in getting ONOS roles" )
+ main.log.error( "Error in getting ONOS roles" )
main.log.warn(
"ONOS1 mastership response: " +
repr( ONOS1Mastership ) )
@@ -958,7 +959,7 @@
ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
intentCheck = main.FALSE
if "Error" in ONOS1Intents or not ONOS1Intents:
- main.log.report( "Error in getting ONOS intents" )
+ main.log.error( "Error in getting ONOS intents" )
main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
else:
intentCheck = main.TRUE
@@ -969,7 +970,7 @@
flowCheck = main.FALSE
ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
if "Error" in ONOS1Flows or not ONOS1Flows:
- main.log.report( "Error in getting ONOS flows" )
+ main.log.error( "Error in getting ONOS flows" )
main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
else:
# TODO: Do a better check, maybe compare flows on switches?
@@ -1121,7 +1122,6 @@
main.log.error( "iCounterValue not defined, setting to 0" )
iCounterValue = 0
- main.log.report( "Restart ONOS node" )
main.case( "Restart ONOS node" )
main.caseExplaination = "Killing ONOS process and restart cli " +\
"sessions once onos is up."
@@ -1154,6 +1154,9 @@
if elapsed:
main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
str( elapsed ) )
+ main.restartTime = elapsed
+ else:
+ main.restartTime = -1
time.sleep( 5 )
# rerun on election apps
main.ONOScli1.electionTestRun()
@@ -1180,14 +1183,12 @@
ONOS1Mastership = main.ONOScli1.roles()
# FIXME: Refactor this whole case for single instance
if "Error" in ONOS1Mastership or not ONOS1Mastership:
- main.log.report( "Error in getting ONOS mastership" )
+ main.log.error( "Error in getting ONOS mastership" )
main.log.warn( "ONOS1 mastership response: " +
repr( ONOS1Mastership ) )
consistentMastership = main.FALSE
else:
consistentMastership = main.TRUE
- main.log.report(
- "Switch roles are consistent across all ONOS nodes" )
utilities.assert_equals(
expect=main.TRUE,
actual=consistentMastership,
@@ -1224,11 +1225,11 @@
ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
intentCheck = main.FALSE
if "Error" in ONOS1Intents or not ONOS1Intents:
- main.log.report( "Error in getting ONOS intents" )
+ main.log.error( "Error in getting ONOS intents" )
main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
else:
intentCheck = main.TRUE
- main.log.report( "Intents are consistent across all ONOS nodes" )
+ main.log.error( "Intents are consistent across all ONOS nodes" )
utilities.assert_equals(
expect=main.TRUE,
actual=intentCheck,
@@ -1253,21 +1254,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
- if intentState and intentState == ONOS1Intents:
+ sameIntents = main.FALSE
+ if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS1 intents: " )
- print json.dumps( json.loads( ONOS1Intents ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) )
- except Exception:
- pass
- sameIntents = main.FALSE
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1313,13 +1334,13 @@
pass
elif leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( "ONOS" + str( controller ) + " sees " +
+ main.log.error( "ONOS" + str( controller ) + " sees " +
str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
@@ -1504,7 +1525,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -1531,7 +1551,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -1557,14 +1576,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -1601,11 +1619,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -1638,9 +1654,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -1691,11 +1705,21 @@
main.step( "Checking ONOS Logs for errors" )
print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
colors[ 'end' ]
- print main.ONOSbench.checkLogs( ONOS1Ip )
+ print main.ONOSbench.checkLogs( ONOS1Ip, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
+
def CASE14( self, main ):
"""
start election app on all onos nodes
@@ -1754,7 +1778,6 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
@@ -1763,7 +1786,7 @@
if leader == ONOS1Ip:
oldLeader = getattr( main, "ONOScli1" )
elif leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -1783,14 +1806,14 @@
main.step( "Make sure new leader is elected" )
leaderN = main.ONOScli1.electionTestLeader()
if leaderN == leader:
- main.log.report( "ONOS still sees " + str( leaderN ) +
+ main.log.error( "ONOS still sees " + str( leaderN ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with electionTestLeader " +
+ main.log.error( "Something is wrong with electionTestLeader " +
"function, check the error logs" )
leaderResult = main.FALSE
elif leaderN is None:
@@ -1868,6 +1891,7 @@
"""
Check for basic functionality with distributed primitives
"""
+ import json
# Make sure variables are defined/set
assert numControllers, "numControllers not defined"
assert main, "main not defined"
@@ -1922,7 +1946,7 @@
# Check that counter incremented numController times
pCounterResults = True
for i in addedPValues:
- tmpResult = i in pCounters
+ tmpResult = i in pCounters
pCounterResults = pCounterResults and tmpResult
if not tmpResult:
main.log.error( str( i ) + " is not in partitioned "
@@ -1992,28 +2016,41 @@
main.step( "Counters we added have the correct values" )
correctResults = main.TRUE
for i in range( numControllers ):
- current = onosCounters[i]
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
try:
- pValue = current.get( pCounterName )
- iValue = current.get( iCounterName )
- if pValue == pCounterValue:
- main.log.info( "Partitioned counter value is correct" )
- else:
- main.log.error( "Partitioned counter value is incorrect," +
- " expected value: " + str( pCounterValue )
- + " current value: " + str( pValue ) )
- correctResults = main.FALSE
- if iValue == iCounterValue:
- main.log.info( "In memory counter value is correct" )
- else:
- main.log.error( "In memory counter value is incorrect, " +
- "expected value: " + str( iCounterValue ) +
- " current value: " + str( iValue ) )
- correctResults = main.FALSE
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
except AttributeError, e:
main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
"is not as expected" )
correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=correctResults,
onpass="Added counters are correct",
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.params b/TestON/tests/SdnIpTest/SdnIpTest.params
index e47adf6..c84af17 100755
--- a/TestON/tests/SdnIpTest/SdnIpTest.params
+++ b/TestON/tests/SdnIpTest/SdnIpTest.params
@@ -16,7 +16,7 @@
<GIT>
<autoPull>on</autoPull>
<branch1>master</branch1>
- <branch2>onos-1.0</branch2>
+ <branch2>onos-1.2</branch2>
</GIT>
<JSON>
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.py b/TestON/tests/SdnIpTest/SdnIpTest.py
index ebbd57e..43bf580 100644
--- a/TestON/tests/SdnIpTest/SdnIpTest.py
+++ b/TestON/tests/SdnIpTest/SdnIpTest.py
@@ -19,6 +19,7 @@
onos-install -f
onos-wait-for-start
"""
+ import time
main.case( "Setting up test environment" )
cellName = main.params[ 'ENV' ][ 'cellName' ]
@@ -47,13 +48,11 @@
# else:
# main.log.warn( "Did not pull new code so skipping mvn " +
# "clean install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
+ cleanInstallResult = main.ONOSbench.cleanInstall( mciTimeout= 1000 )
main.ONOSbench.getVersion( report=True )
- #cellResult = main.ONOSbench.setCell( cellName )
- #verifyResult = main.ONOSbench.verifyCell()
main.step( "Creating ONOS package" )
- packageResult = main.ONOSbench.onosPackage()
+ packageResult = main.ONOSbench.onosPackage( opTimeout=500 )
main.step( "Installing ONOS package" )
onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
@@ -61,13 +60,14 @@
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
- onos1Isup = main.ONOSbench.isup( ONOS1Ip )
+ onos1Isup = main.ONOSbench.isup( ONOS1Ip, timeout=420 )
if onos1Isup:
break
if not onos1Isup:
main.log.report( "ONOS1 didn't start!" )
- cliResult = main.ONOScli.startOnosCli( ONOS1Ip )
+ cliResult = main.ONOScli.startOnosCli( ONOS1Ip,
+ commandlineTimeout=100, onosStartTimeout=600)
case1Result = ( cleanInstallResult and packageResult and
cellResult and verifyResult and
@@ -145,8 +145,271 @@
listResult = main.ONOScli.devices( jsonFormat=False )
main.log.info( listResult )
time.sleep( 10 )
- main.log.info( "Installing sdn-ip feature" )
- main.ONOScli.featureInstall( "onos-app-sdnip" )
+ main.log.info( "Activate sdn-ip application" )
+ main.ONOScli.activateApp( "org.onosproject.sdnip" )
+ time.sleep( 10 )
+ main.step( "Login all BGP peers and add routes into peers" )
+
+ main.log.info( "Login Quagga CLI on host3" )
+ main.QuaggaCliHost3.loginQuagga( "1.168.30.2" )
+ main.log.info( "Enter configuration model of Quagga CLI on host3" )
+ main.QuaggaCliHost3.enterConfig( 64514 )
+ main.log.info( "Add routes to Quagga on host3" )
+ main.QuaggaCliHost3.addRoutes( prefixesHost3, 1 )
+
+ main.log.info( "Login Quagga CLI on host4" )
+ main.QuaggaCliHost4.loginQuagga( "1.168.30.3" )
+ main.log.info( "Enter configuration model of Quagga CLI on host4" )
+ main.QuaggaCliHost4.enterConfig( 64516 )
+ main.log.info( "Add routes to Quagga on host4" )
+ main.QuaggaCliHost4.addRoutes( prefixesHost4, 1 )
+
+ main.log.info( "Login Quagga CLI on host5" )
+ main.QuaggaCliHost5.loginQuagga( "1.168.30.5" )
+ main.log.info( "Enter configuration model of Quagga CLI on host5" )
+ main.QuaggaCliHost5.enterConfig( 64521 )
+ main.log.info( "Add routes to Quagga on host5" )
+ main.QuaggaCliHost5.addRoutes( prefixesHost5, 1 )
+
+ for i in range( 101, 201 ):
+ prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
+ main.log.info( prefixesHostX )
+ for prefix in prefixesHostX:
+ allRoutesExpected.append( prefix + "/" + "192.168.40."
+ + str( i - 100 ) )
+
+ routeIntentsExpectedHostX = \
+ main.QuaggaCliHost.generateExpectedOnePeerRouteIntents(
+ prefixesHostX, "192.168.40." + str( i - 100 ),
+ "00:00:%02d:00:00:90" % ( i - 101 ), SDNIPJSONFILEPATH )
+ routeIntentsExpected = routeIntentsExpected + \
+ routeIntentsExpectedHostX
+
+ main.log.info( "Login Quagga CLI on host" + str( i ) )
+ QuaggaCliHostX = getattr( main, ( 'QuaggaCliHost' + str( i ) ) )
+ QuaggaCliHostX.loginQuagga( "1.168.30." + str( i ) )
+ main.log.info(
+ "Enter configuration model of Quagga CLI on host" + str( i ) )
+ QuaggaCliHostX.enterConfig( 65000 + i - 100 )
+ main.log.info( "Add routes to Quagga on host" + str( i ) )
+ QuaggaCliHostX.addRoutes( prefixesHostX, 1 )
+
+ time.sleep( 60 )
+ # get routes inside SDN-IP
+ getRoutesResult = main.ONOScli.routes( jsonFormat=True )
+
+ allRoutesActual = \
+ main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+
+ allRoutesStrExpected = str( sorted( allRoutesExpected ) )
+ allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )
+ main.step( "Check routes installed" )
+ main.log.info( "Routes expected:" )
+ main.log.info( allRoutesStrExpected )
+ main.log.info( "Routes get from ONOS CLI:" )
+ main.log.info( allRoutesStrActual )
+ utilities.assertEquals(
+ expect=allRoutesStrExpected, actual=allRoutesStrActual,
+ onpass="***Routes in SDN-IP are correct!***",
+ onfail="***Routes in SDN-IP are wrong!***" )
+ if( eq( allRoutesStrExpected, allRoutesStrActual ) ):
+ main.log.report(
+ "***Routes in SDN-IP after adding routes are correct!***" )
+ else:
+ main.log.report(
+ "***Routes in SDN-IP after adding routes are wrong!***" )
+
+ time.sleep( 20 )
+ getIntentsResult = main.ONOScli.intents( jsonFormat=True )
+
+ main.step( "Check MultiPointToSinglePointIntent intents installed" )
+ # routeIntentsExpected are generated when generating routes
+ # get route intents from ONOS CLI
+ routeIntentsActualNum = \
+ main.QuaggaCliHost3.extractActualRouteIntentNum( getIntentsResult )
+ routeIntentsExpectedNum = 1030
+ main.log.info( "MultiPointToSinglePoint Intent Num expected is:" )
+ main.log.info( routeIntentsExpectedNum )
+ main.log.info( "MultiPointToSinglePoint Intent NUM Actual is:" )
+ main.log.info( routeIntentsActualNum )
+ utilities.assertEquals(
+ expect=True,
+ actual=eq( routeIntentsExpectedNum, routeIntentsActualNum ),
+ onpass="***MultiPointToSinglePoint Intent Num in SDN-IP is \
+ correct!***",
+ onfail="***MultiPointToSinglePoint Intent Num in SDN-IP is \
+ wrong!***" )
+
+ if( eq( routeIntentsExpectedNum, routeIntentsActualNum ) ):
+ main.log.report( "***MultiPointToSinglePoint Intents before \
+ deleting routes correct!***" )
+ else:
+ main.log.report( "***MultiPointToSinglePoint Intents before \
+ deleting routes wrong!***" )
+
+ main.step( "Check BGP PointToPointIntent intents installed" )
+
+ bgpIntentsActualNum = \
+ main.QuaggaCliHost3.extractActualBgpIntentNum( getIntentsResult )
+ bgpIntentsExpectedNum = 624
+ main.log.info( "bgpIntentsExpected num is:" )
+ main.log.info( bgpIntentsExpectedNum )
+ main.log.info( "bgpIntentsActual num is:" )
+ main.log.info( bgpIntentsActualNum)
+ utilities.assertEquals(
+ expect=True,
+ actual=eq( bgpIntentsExpectedNum, bgpIntentsActualNum ),
+ onpass="***PointToPointIntent Intent Num in SDN-IP are correct!***",
+ onfail="***PointToPointIntent Intent Num in SDN-IP are wrong!***" )
+ if ( eq( bgpIntentsExpectedNum, bgpIntentsActualNum ) ):
+ main.log.report(
+ "***PointToPointIntent Intents in SDN-IP are correct!***" )
+ else:
+ main.log.report(
+ "***PointToPointIntent Intents in SDN-IP are wrong!***" )
+
+ #============================= Ping Test ========================
+ # Wait until all MultiPointToSinglePoint intents are in system
+ time.sleep( 20 )
+ pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
+ pingTestResultsFile = \
+ "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-before-delete-routes-"\
+ + strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
+ pingTestResults = main.QuaggaCliHost.pingTest(
+ "1.168.30.100", pingTestScript, pingTestResultsFile )
+ main.log.info( pingTestResults )
+ time.sleep( 20 )
+
+ #============================= Deleting Routes ==================
+ main.step( "Check deleting routes installed" )
+ main.QuaggaCliHost3.deleteRoutes( prefixesHost3, 1 )
+ main.QuaggaCliHost4.deleteRoutes( prefixesHost4, 1 )
+ main.QuaggaCliHost5.deleteRoutes( prefixesHost5, 1 )
+
+ for i in range( 101, 201 ):
+ prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
+ main.log.info( prefixesHostX )
+ QuaggaCliHostX = getattr( main, ( 'QuaggaCliHost' + str( i ) ) )
+ QuaggaCliHostX.deleteRoutes( prefixesHostX, 1 )
+
+ getRoutesResult = main.ONOScli.routes( jsonFormat=True )
+ allRoutesActual = \
+ main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+ main.log.info( "allRoutes_actual = " )
+ main.log.info( allRoutesActual )
+
+ utilities.assertEquals(
+ expect="[]", actual=str( allRoutesActual ),
+ onpass="***Route number in SDN-IP is 0, correct!***",
+ onfail="***Routes number in SDN-IP is not 0, wrong!***" )
+
+ if( eq( allRoutesStrExpected, allRoutesStrActual ) ):
+ main.log.report( "***Routes in SDN-IP after deleting correct!***" )
+ else:
+ main.log.report( "***Routes in SDN-IP after deleting wrong!***" )
+
+ main.step( "Check intents after deleting routes" )
+ getIntentsResult = main.ONOScli.intents( jsonFormat=True )
+ routeIntentsActualNum = \
+ main.QuaggaCliHost3.extractActualRouteIntentNum(
+ getIntentsResult )
+ main.log.info( "route Intents Actual Num is: " )
+ main.log.info( routeIntentsActualNum )
+ utilities.assertEquals(
+ expect=0, actual=routeIntentsActualNum,
+ onpass="***MultiPointToSinglePoint Intent Num in SDN-IP is 0, \
+ correct!***",
+ onfail="***MultiPointToSinglePoint Intent Num in SDN-IP is not 0, \
+ wrong!***" )
+
+ if( eq( 0, routeIntentsActualNum ) ):
+ main.log.report( "***MultiPointToSinglePoint Intents after \
+ deleting routes correct!***" )
+ else:
+ main.log.report( "***MultiPointToSinglePoint Intents after \
+ deleting routes wrong!***" )
+
+ time.sleep( 20 )
+ pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
+ pingTestResultsFile = \
+ "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-"\
+ + strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
+ pingTestResults = main.QuaggaCliHost.pingTest(
+ "1.168.30.100", pingTestScript, pingTestResultsFile )
+ main.log.info( pingTestResults )
+ time.sleep( 100 )
+
+ def CASE_4_origin( self, main ):
+ """
+ Test the SDN-IP functionality
+ allRoutesExpected: all expected routes for all BGP peers
+ routeIntentsExpected: all expected MultiPointToSinglePointIntent \
+ intents
+ bgpIntentsExpected: expected PointToPointIntent intents
+ allRoutesActual: all routes from ONOS LCI
+ routeIntentsActual: actual MultiPointToSinglePointIntent intents from \
+ ONOS CLI
+ bgpIntentsActual: actual PointToPointIntent intents from ONOS CLI
+ """
+ import time
+ import json
+ from operator import eq
+ from time import localtime, strftime
+
+ main.case("This case is to testing the functionality of SDN-IP with \
+ single ONOS instance" )
+ SDNIPJSONFILEPATH = \
+ "/home/admin/ONOS/tools/package/config/sdnip.json"
+ # all expected routes for all BGP peers
+ allRoutesExpected = []
+ main.step( "Start to generate routes for all BGP peers" )
+ main.log.info( "Generate prefixes for host3" )
+ prefixesHost3 = main.QuaggaCliHost3.generatePrefixes( 3, 10 )
+ main.log.info( prefixesHost3 )
+ # generate route with next hop
+ for prefix in prefixesHost3:
+ allRoutesExpected.append( prefix + "/" + "192.168.20.1" )
+ routeIntentsExpectedHost3 = \
+ main.QuaggaCliHost3.generateExpectedOnePeerRouteIntents(
+ prefixesHost3, "192.168.20.1", "00:00:00:00:02:02",
+ SDNIPJSONFILEPATH )
+
+ main.log.info( "Generate prefixes for host4" )
+ prefixesHost4 = main.QuaggaCliHost4.generatePrefixes( 4, 10 )
+ main.log.info( prefixesHost4 )
+ # generate route with next hop
+ for prefix in prefixesHost4:
+ allRoutesExpected.append( prefix + "/" + "192.168.30.1" )
+ routeIntentsExpectedHost4 = \
+ main.QuaggaCliHost4.generateExpectedOnePeerRouteIntents(
+ prefixesHost4, "192.168.30.1", "00:00:00:00:03:01",
+ SDNIPJSONFILEPATH )
+
+ main.log.info( "Generate prefixes for host5" )
+ prefixesHost5 = main.QuaggaCliHost5.generatePrefixes( 5, 10 )
+ main.log.info( prefixesHost5 )
+ for prefix in prefixesHost5:
+ allRoutesExpected.append( prefix + "/" + "192.168.60.2" )
+ routeIntentsExpectedHost5 = \
+ main.QuaggaCliHost5.generateExpectedOnePeerRouteIntents(
+ prefixesHost5, "192.168.60.1", "00:00:00:00:06:02",
+ SDNIPJSONFILEPATH )
+
+ routeIntentsExpected = routeIntentsExpectedHost3 + \
+ routeIntentsExpectedHost4 + routeIntentsExpectedHost5
+
+ main.step( "Get devices in the network" )
+ listResult = main.ONOScli.devices( jsonFormat=False )
+ main.log.info( listResult )
+ time.sleep( 10 )
+ if branchName == "onos-1.1":
+ main.log.info( "Installing sdn-ip feature" )
+ main.ONOScli.featureInstall( "onos-app-sdnip" )
+ else:
+ main.log.info( "Activate sdn-ip application" )
+ main.ONOScli.activateApp( "org.onosproject.sdnip" )
+ #main.log.info( "Installing sdn-ip feature" )
+ #main.ONOScli.featureInstall( "onos-app-sdnip" )
time.sleep( 10 )
main.step( "Login all BGP peers and add routes into peers" )
@@ -199,15 +462,18 @@
getRoutesResult = main.ONOScli.routes( jsonFormat=True )
# parse routes from ONOS CLI
- if branchName == "master":
- allRoutesActual = \
+ #if branchName == "master":
+ # allRoutesActual = \
+ # main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+ #elif branchName == "onos-1.0":
+ # allRoutesActual = \
+ # main.QuaggaCliHost3.extractActualRoutesOneDotZero( getRoutesResult )
+ #else:
+ # main.log("ONOS is on wrong branch")
+ # exit
+
+ allRoutesActual = \
main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
- elif branchName == "onos-1.0":
- allRoutesActual = \
- main.QuaggaCliHost3.extractActualRoutesOneDotZero( getRoutesResult )
- else:
- main.log("ONOS is on wrong branch")
- exit
allRoutesStrExpected = str( sorted( allRoutesExpected ) )
allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )
@@ -436,8 +702,12 @@
listResult = main.ONOScli.devices( jsonFormat=False )
main.log.info( listResult )
time.sleep( 10 )
- main.log.info( "Installing sdn-ip feature" )
- main.ONOScli.featureInstall( "onos-app-sdnip" )
+ if branchName == "onos-1.1":
+ main.log.info( "Installing sdn-ip feature" )
+ main.ONOScli.featureInstall( "onos-app-sdnip" )
+ else:
+ main.log.info( "Activate sdn-ip application" )
+ main.ONOScli.activateApp( "org.onosproject.sdnip" )
time.sleep( 10 )
main.step( "Login all BGP peers and add routes into peers" )