Merge pull request #98 from opennetworkinglab/devl/HA_refactor_threads
Devl/ha refactor threads
diff --git a/TestON/bin/cli.py b/TestON/bin/cli.py
index 47b027e..ed1c796 100755
--- a/TestON/bin/cli.py
+++ b/TestON/bin/cli.py
@@ -48,11 +48,12 @@
dump = pprint.PrettyPrinter(indent=4)
__builtin__.testthread = False
introduction = "TestON is the testing framework \nDeveloped by Paxterra Solutions (www.paxterrasolutions.com)"
+__builtin__.COLORS = False
path = re.sub("teston$", "", os.getcwd())
sys.path.append(path+"/Core")
sys.path.append("../")
-from core.teston import *
+from core.teston import *
class CLI( threading.Thread,Cmd,object ):
"command-line interface to execute the test."
@@ -61,7 +62,7 @@
def __init__( self, teston, stdin=sys.stdin ):
self.teston = teston
-
+
self._mainevent = threading.Event()
threading.Thread.__init__(self)
self.main_stop = False
@@ -592,6 +593,8 @@
if __name__ == '__main__':
if len(sys.argv) > 1:
+ __builtin__.COLORS = True
CLI("test").onecmd(' '.join(sys.argv[1:]))
else:
+ __builtin__.COLORS = False
CLI("test").cmdloop()
diff --git a/TestON/core/Thread.py b/TestON/core/Thread.py
index e20abc3..da30339 100644
--- a/TestON/core/Thread.py
+++ b/TestON/core/Thread.py
@@ -1,9 +1,11 @@
#!/usr/bin/env python
import threading
-class Thread(threading.Thread):
- def __init__(self, target = None, threadID=None, name="", args=(), kwargs={}):
- super(Thread, self).__init__()
+
+class Thread( threading.Thread ):
+ def __init__( self, target=None, threadID=None, name="", args=(),
+ kwargs={} ):
+ super( Thread, self ).__init__()
self.threadID = threadID
self.name = name
self.target = target
@@ -14,11 +16,8 @@
def run( self ):
try:
if self.target is not None:
- if len(self.args) != 0:
- self.result = self.target( *self.args )
- else:
- self.result = self.target()
+ self.result = self.target( *self.args, **self.kwargs )
except Exception as e:
- print "Thread-" + str(self.threadID) + \
- ":something went wrong with " + self.name + " method"
+ print "Thread-" + str( self.threadID ) + " '" + self.name + "'"\
+ ":something went wrong with " + self.target + " method"
print e
diff --git a/TestON/core/logger.py b/TestON/core/logger.py
index 80fe0de..1dfe6bf 100644
--- a/TestON/core/logger.py
+++ b/TestON/core/logger.py
@@ -115,8 +115,8 @@
#### Add log-level - Report
logging.addLevelName(9, "REPORT")
logging.addLevelName(7, "EXACT")
- logging.addLevelName(10, "CASE")
- logging.addLevelName(11, "STEP")
+ logging.addLevelName(11, "CASE")
+ logging.addLevelName(12, "STEP")
main.log = logging.getLogger(main.TEST)
def report (msg):
'''
@@ -183,13 +183,42 @@
### initializing logging module and settig log level
main.log.setLevel(logging.INFO)
+ main.log.setLevel(logging.DEBUG) # Temporary
main.LogFileHandler.setLevel(logging.INFO)
# create console handler with a higher log level
main.ConsoleHandler = logging.StreamHandler()
main.ConsoleHandler.setLevel(logging.INFO)
+ main.ConsoleHandler.setLevel(logging.DEBUG) #Temporary
# create formatter and add it to the handlers
- formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ #formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+ class MyFormatter( logging.Formatter ):
+ colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
+ 'blue': '\033[94m', 'green': '\033[92m',
+ 'yellow': '\033[93m', 'red': '\033[91m',
+ 'end': '\033[0m' }
+
+ FORMATS = {'DEFAULT': '%(asctime)s - %(name)s - %(levelname)s - %(message)s'}
+ if COLORS: # NOTE:colors will only be loaded if command is run from one line
+ # IE: './cli.py run testname'
+ # This is to prevent issues with Jenkins parsing
+ # TODO: Make colors configurable
+ levels = { logging.ERROR : colors['red'] +
+ FORMATS['DEFAULT'] +
+ colors['end'],
+ logging.WARN : colors['yellow'] +
+ FORMATS['DEFAULT'] +
+ colors['end'],
+ logging.DEBUG : colors['purple'] +
+ FORMATS['DEFAULT'] +
+ colors['end'] }
+ FORMATS.update( levels )
+
+ def format( self, record ):
+ self._fmt = self.FORMATS.get( record.levelno,
+ self.FORMATS[ 'DEFAULT' ] )
+ return logging.Formatter.format( self, record )
+ formatter = MyFormatter()
main.ConsoleHandler.setFormatter(formatter)
main.LogFileHandler.setFormatter(formatter)
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index 6e95e3e..edb763c 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -1,7 +1,7 @@
#!/usr/bin/env python
'''
Created on 22-Oct-2012
-
+
@author: Anil Kumar (anilkumar.s@paxterrasolutions.com)
@@ -16,7 +16,7 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with TestON. If not, see <http://www.gnu.org/licenses/>.
+ along with TestON. If not, see <http://www.gnu.org/licenses/>.
@@ -90,7 +90,7 @@
self.logs_path = logs_path
self.driver = ''
self.Thread = Thread
-
+
self.configparser()
verifyOptions(options)
load_logger()
@@ -99,17 +99,17 @@
self.driversList=[]
if type(self.componentDictionary) == str :
self.componentDictionary = dict(self.componentDictionary)
-
+
for component in self.componentDictionary :
self.driversList.append(self.componentDictionary[component]['type'])
-
+
self.driversList = list(set(self.driversList)) # Removing duplicates.
# Checking the test_target option set for the component or not
if type(self.componentDictionary) == dict:
for component in self.componentDictionary.keys():
if 'test_target' in self.componentDictionary[component].keys():
self.test_target = component
-
+
# Checking for the openspeak file and test script
self.logger.initlog(self)
diff --git a/TestON/dependencies/Jenkins_getresult_HA.py b/TestON/dependencies/Jenkins_getresult_HA.py
index 9cb3b8b..7b4fc09 100755
--- a/TestON/dependencies/Jenkins_getresult_HA.py
+++ b/TestON/dependencies/Jenkins_getresult_HA.py
@@ -70,7 +70,7 @@
output +="<p></p>"
#output +=" Date: %s, %s %s" % (name.split("_")[2], name.split("_")[1], name.split("_")[3]) + "<p>*******************<p>"
- #Open the latest log folder
+ #Open the latest log folder
output += "<h2>Test "+str(test)+"</h2><p>************************************</p>"
f = open(path + name + ".rpt")
@@ -102,8 +102,11 @@
#https://wiki.onosproject.org/display/OST/Test+Results+-+HA#Test+Results+-+HA
#Example anchor on new wiki: #TestResults-HA-TestHATestSanity
page_name = "Master-HA"
- if "ONOS-HA-Maint" in job:
- #NOTE if page name starts with number it prepends 'id-' to anchor links
+ if "ONOS-HA-1.1.X" in job:
+ page_name = "Blackbird-HA"
+ elif "ONOS-HA-Maint" in job:
+ # NOTE if page name starts with number confluence prepends 'id-'
+ # to anchor links
page_name = "id-1.0-HA"
header += "<li><a href=\'#" + str(page_name) + "-Test" + str(test) + "\'> " + str(test) + " - Results: " + str(passes) + " Passed, " + str(fails) + " Failed</a></li>"
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index dc9de05..a6436ca 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -326,7 +326,7 @@
command = args[ "SRC" ] + " ping " + \
args[ "TARGET" ] + " -c 1 -i 1 -W 8"
try:
- main.log.warn( "Sending: " + command )
+ main.log.info( "Sending: " + command )
self.handle.sendline( command )
i = self.handle.expect( [ command, pexpect.TIMEOUT ] )
if i == 1:
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index 8e445a9..e451335 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -299,7 +299,7 @@
self.handle.sendline( "" )
i = self.handle.expect( [ 'No\ssuch\device', 'listening\son',
pexpect.TIMEOUT, "\$" ], timeout=10 )
- main.log.warn( self.handle.before + self.handle.after )
+ main.log.info( self.handle.before + self.handle.after )
if i == 0:
main.log.error( self.name + ": tcpdump - No such device exists.\
tcpdump attempted on: " + intf )
@@ -591,7 +591,6 @@
if actionType == 'remove':
# -D is the 'delete' rule of iptables
actionRemove = '-D'
- # noinspection PyBroadException
try:
self.handle.sendline( "" )
# Delete a specific rule specified into the function
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 419da23..04d2eed 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -49,7 +49,7 @@
self.home = self.options[ 'home' ]
break
if self.home is None or self.home == "":
- self.home = "~/ONOS"
+ self.home = "~/onos"
self.name = self.options[ 'name' ]
self.handle = super( OnosCliDriver, self ).connect(
@@ -84,19 +84,23 @@
Called when Test is complete to disconnect the ONOS handle.
"""
response = main.TRUE
- # noinspection PyBroadException
try:
- self.logout()
- self.handle.sendline( "" )
- self.handle.expect( "\$" )
- self.handle.sendline( "exit" )
- self.handle.expect( "closed" )
+ if self.handle:
+ i = self.logout()
+ if i == main.TRUE:
+ self.handle.sendline( "" )
+ self.handle.expect( "\$" )
+ self.handle.sendline( "exit" )
+ self.handle.expect( "closed" )
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
response = main.FALSE
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
+ except ValueError:
+ main.log.exception( "Exception in discconect of " + self.name )
+ response = main.TRUE
except Exception:
main.log.exception( self.name + ": Connection failed to the host" )
response = main.FALSE
@@ -105,27 +109,36 @@
def logout( self ):
"""
Sends 'logout' command to ONOS cli
+ Returns main.TRUE if exited CLI and
+ main.FALSE on timeout (not guranteed you are disconnected)
+ None on TypeError
+ Exits test on unknown error or pexpect exits unexpectedly
"""
try:
- self.handle.sendline( "" )
- i = self.handle.expect( [ "onos>", "\$", pexpect.TIMEOUT ],
- timeout=10 )
- if i == 0: # In ONOS CLI
- self.handle.sendline( "logout" )
- self.handle.expect( "\$" )
- elif i == 1: # not in CLI
+ if self.handle:
+ self.handle.sendline( "" )
+ i = self.handle.expect( [ "onos>", "\$", pexpect.TIMEOUT ],
+ timeout=10 )
+ if i == 0: # In ONOS CLI
+ self.handle.sendline( "logout" )
+ self.handle.expect( "\$" )
+ return main.TRUE
+ elif i == 1: # not in CLI
+ return main.TRUE
+ elif i == 3: # Timeout
+ return main.FALSE
+ else:
return main.TRUE
- elif i == 3: # Timeout
- return main.FALSE
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
return None
except pexpect.EOF:
main.log.error( self.name + ": eof exception found" )
- main.log.error( self.name + ": " +
- self.handle.before )
+ main.log.error( self.name + ": " + self.handle.before )
main.cleanup()
main.exit()
+ except ValueError:
+ main.log.error( self.name + "ValueError exception in logout method" )
except Exception:
main.log.exception( self.name + ": Uncaught exception!" )
main.cleanup()
@@ -349,7 +362,6 @@
Optional:
* tcpPort
"""
- # noinspection PyBroadException
try:
cmdStr = "add-node " + str( nodeId ) + " " +\
str( ONOSIp ) + " " + str( tcpPort )
@@ -402,16 +414,24 @@
main.cleanup()
main.exit()
- def nodes( self ):
+ def nodes( self, jsonFormat=True):
"""
List the nodes currently visible
Issues command: 'nodes'
- Returns: entire handle of list of nodes
+ Optional argument:
+ * jsonFormat - boolean indicating if you want output in json
"""
try:
- cmdStr = "nodes"
- handle = self.sendline( cmdStr )
- return handle
+ if jsonFormat:
+ cmdStr = "nodes -j"
+ output = self.sendline( cmdStr )
+ ansiEscape = re.compile( r'\r\r\n\x1b[^m]*m' )
+ parsedOutput = ansiEscape.sub( '', output )
+ return parsedOutput
+ else:
+ cmdStr = "nodes"
+ output = self.sendline( cmdStr )
+ return output
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
return None
@@ -966,6 +986,7 @@
handle = self.sendline( cmdStr )
if re.search( "Error", handle ):
main.log.error( "Error in adding Host intent" )
+ main.log.debug( "Response from ONOS was: " + repr( handle ) )
return None
else:
main.log.info( "Host intent installed between " +
@@ -975,6 +996,8 @@
return match.group()[3:-1]
else:
main.log.error( "Error, intent ID not found" )
+ main.log.debug( "Response from ONOS was: " +
+ repr( handle ) )
return None
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
@@ -1318,10 +1341,10 @@
purge=False, sync=False ):
"""
Remove intent for specified application id and intent id
- Optional args:-
+ Optional args:-
-s or --sync: Waits for the removal before returning
- -p or --purge: Purge the intent from the store after removal
-
+ -p or --purge: Purge the intent from the store after removal
+
Returns:
main.False on error and
cli output otherwise
@@ -1491,7 +1514,7 @@
else:
cmdStr = "flows"
handle = self.sendline( cmdStr )
- if re.search( "Error\sexecuting\scommand:", handle ):
+ if re.search( "Error:", handle ):
main.log.error( self.name + ".flows() response: " +
str( handle ) )
return handle
@@ -2194,15 +2217,31 @@
main.cleanup()
main.exit()
- def leaders( self ):
+ def leaders( self, jsonFormat=True ):
"""
Returns the output of the leaders command.
+ Optional argument:
+ * jsonFormat - boolean indicating if you want output in json
"""
# FIXME: add json output
+ # Sample JSON
+ # {
+ # "electedTime": "13m ago",
+ # "epoch": 4,
+ # "leader": "10.128.30.17",
+ # "topic": "intent-partition-3"
+ # },
try:
- output = self.sendline( "onos:leaders" )
- main.log.warn( output )
- return output
+ if jsonFormat:
+ cmdStr = "onos:leaders -j"
+ output = self.sendline( cmdStr )
+ ansiEscape = re.compile( r'\r\r\n\x1b[^m]*m' )
+ cleanedOutput = ansiEscape.sub( '', output )
+ return cleanedOutput
+ else:
+ cmdStr = "onos:leaders"
+ output = self.sendline( cmdStr )
+ return output
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
return None
@@ -2216,15 +2255,21 @@
main.cleanup()
main.exit()
- def pendingMap( self ):
+ def pendingMap( self, jsonFormat=True ):
"""
Returns the output of the intent Pending map.
"""
- # FIXME: add json output
try:
- output = self.sendline( "onos:intents -p" )
- main.log.warn( output )
- return output
+ if jsonFormat:
+ cmdStr = "onos:intents -p -j"
+ output = self.sendline( cmdStr )
+ ansiEscape = re.compile( r'\r\r\n\x1b[^m]*m' )
+ cleanedOutput = ansiEscape.sub( '', output )
+ return cleanedOutput
+ else:
+ cmdStr = "onos:intents -p"
+ output = self.sendline( cmdStr )
+ return output
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
return None
@@ -2238,15 +2283,32 @@
main.cleanup()
main.exit()
- def partitions( self ):
+ def partitions( self, jsonFormat=True ):
"""
Returns the output of the raft partitions command for ONOS.
"""
- # FIXME: add json output
+ # Sample JSON
+ # {
+ # "leader": "tcp://10.128.30.11:7238",
+ # "members": [
+ # "tcp://10.128.30.11:7238",
+ # "tcp://10.128.30.17:7238",
+ # "tcp://10.128.30.13:7238",
+ # ],
+ # "name": "p1",
+ # "term": 3
+ # },
try:
- output = self.sendline( "partitions" )
- main.log.warn( output )
- return output
+ if jsonFormat:
+ cmdStr = "onos:partitions -j"
+ output = self.sendline( cmdStr )
+ ansiEscape = re.compile( r'\r\r\n\x1b[^m]*m' )
+ cleanedOutput = ansiEscape.sub( '', output )
+ return cleanedOutput
+ else:
+ cmdStr = "onos:partitions"
+ output = self.sendline( cmdStr )
+ return output
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
return None
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index b08054b..14562e7 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -48,7 +48,7 @@
self.home = self.options[ 'home' ]
break
if self.home is None or self.home == "":
- self.home = "~/ONOS"
+ self.home = "~/onos"
self.name = self.options[ 'name' ]
self.handle = super( OnosDriver, self ).connect(
@@ -81,13 +81,17 @@
"""
response = main.TRUE
try:
- self.handle.sendline( "" )
- self.handle.expect( "\$" )
- self.handle.sendline( "exit" )
- self.handle.expect( "closed" )
+ if self.handle:
+ self.handle.sendline( "" )
+ self.handle.expect( "\$" )
+ self.handle.sendline( "exit" )
+ self.handle.expect( "closed" )
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
+ except ValueError:
+ main.log.exception( "Exception in discconect of " + self.name )
+ response = main.TRUE
except Exception:
main.log.exception( self.name + ": Connection failed to the host" )
response = main.FALSE
@@ -173,6 +177,7 @@
'Runtime\sEnvironment\sto\scontinue',
'BUILD\sFAILURE',
'BUILD\sSUCCESS',
+ 'onos\$', #TODO: fix this to be more generic?
'ONOS\$',
pexpect.TIMEOUT ], timeout=600 )
if i == 0:
@@ -188,7 +193,7 @@
main.exit()
elif i == 2:
main.log.info( self.name + ": Build success!" )
- elif i == 3:
+ elif i == 3 or i == 4:
main.log.info( self.name + ": Build complete" )
# Print the build time
for line in self.handle.before.splitlines():
@@ -197,7 +202,7 @@
self.handle.sendline( "" )
self.handle.expect( "\$", timeout=60 )
return main.TRUE
- elif i == 4:
+ elif i == 5:
main.log.error(
self.name +
": mvn clean install TIMEOUT!" )
@@ -220,10 +225,14 @@
main.cleanup()
main.exit()
- def gitPull( self, comp1="" ):
+ def gitPull( self, comp1="", fastForward=True ):
"""
Assumes that "git pull" works without login
+ If the fastForward boolean is set to true, only git pulls that can
+ be fast forwarded will be performed. IE if you have not local commits
+ in your branch.
+
This function will perform a git pull on the ONOS instance.
If used as gitPull( "NODE" ) it will do git pull + NODE. This is
for the purpose of pulling from other nodes if necessary.
@@ -239,11 +248,12 @@
# self.stop()
self.handle.sendline( "cd " + self.home )
self.handle.expect( self.home + "\$" )
- if comp1 == "":
- self.handle.sendline( "git pull" )
- else:
- self.handle.sendline( "git pull " + comp1 )
-
+ cmd = "git pull"
+ if comp1 != "":
+ cmd += ' ' + comp1
+ if fastForward:
+ cmd += ' ' + " --ff-only"
+ self.handle.sendline( cmd )
i = self.handle.expect(
[
'fatal',
@@ -254,6 +264,9 @@
'You\sare\snot\scurrently\son\sa\sbranch',
'You asked me to pull without telling me which branch you',
'Pull is not possible because you have unmerged files',
+ 'Please enter a commit message to explain why this merge',
+ 'Found a swap file by the name',
+ 'Please, commit your changes before you can merge.',
pexpect.TIMEOUT ],
timeout=300 )
# debug
@@ -261,7 +274,11 @@
# "git pull response: " +
# str( self.handle.before ) + str( self.handle.after ) )
if i == 0:
- main.log.error( self.name + ": Git pull had some issue..." )
+ main.log.error( self.name + ": Git pull had some issue" )
+ output = self.handle.after
+ self.handle.expect( '\$' )
+ output += self.handle.before
+ main.log.warn( output )
return main.ERROR
elif i == 1:
main.log.error(
@@ -303,6 +320,29 @@
"you have unmerged files." )
return main.ERROR
elif i == 8:
+ # NOTE: abandoning test since we can't reliably handle this
+ # there could be different default text editors and we
+ # also don't know if we actually want to make the commit
+ main.log.error( "Git pull resulted in a merge commit message" +
+ ". Exiting test!" )
+ main.cleanup()
+ main.exit()
+ elif i == 9: # Merge commit message but swap file exists
+ main.log.error( "Git pull resulted in a merge commit message" +
+ " but a swap file exists." )
+ try:
+ self.handle.send( 'A' ) # Abort
+ self.handle.expect( "\$" )
+ return main.ERROR
+ except Exception:
+ main.log.exception( "Couldn't exit editor prompt!")
+ main.cleanup()
+ main.exit()
+ elif i == 10: # In the middle of a merge commit
+ main.log.error( "Git branch is in the middle of a merge. " )
+ main.log.warn( self.handle.before + self.handle.after )
+ return main.ERROR
+ elif i == 11:
main.log.error( self.name + ": Git Pull - TIMEOUT" )
main.log.error(
self.name + " Response was: " + str(
@@ -348,9 +388,9 @@
self.handle.expect( cmd )
i = self.handle.expect(
[ 'fatal',
- 'Username\sfor\s(.*):\s',
- 'Already\son\s\'',
- 'Switched\sto\sbranch\s\'' + str( branch ),
+ 'Username for (.*): ',
+ 'Already on \'',
+ 'Switched to branch \'' + str( branch ),
pexpect.TIMEOUT,
'error: Your local changes to the following files' +
'would be overwritten by checkout:',
@@ -805,6 +845,7 @@
" stop" )
i = self.handle.expect( [
"stop/waiting",
+ "Could not resolve hostname",
"Unknown\sinstance",
pexpect.TIMEOUT ], timeout=60 )
@@ -812,9 +853,12 @@
main.log.info( "ONOS service stopped" )
return main.TRUE
elif i == 1:
- main.log.info( "Unknown ONOS instance specified: " +
+ main.log.info( "onosStop() Unknown ONOS instance specified: " +
str( nodeIp ) )
return main.FALSE
+ elif i == 2:
+ main.log.warn( "ONOS wasn't running" )
+ return main.TRUE
else:
main.log.error( "ONOS service failed to stop" )
return main.FALSE
@@ -1224,14 +1268,14 @@
def runOnosTopoCfg( self, instanceName, jsonFile ):
"""
On ONOS bench, run this command:
- ./~/ONOS/tools/test/bin/onos-topo-cfg $OC1 filename
+ {ONOS_HOME}/tools/test/bin/onos-topo-cfg $OC1 filename
which starts the rest and copies
the json file to the onos instance
"""
try:
self.handle.sendline( "" )
self.handle.expect( "\$" )
- self.handle.sendline( "cd ~/ONOS/tools/test/bin" )
+ self.handle.sendline( "cd " + self.home + "/tools/test/bin" )
self.handle.expect( "/bin$" )
cmd = "./onos-topo-cfg " + instanceName + " " + jsonFile
print "cmd = ", cmd
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 6655f33..0bd05ad 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -49,7 +49,7 @@
main.case( "Setting up test environment" )
# TODO: save all the timers and output them for plotting
- # load some vairables from the params file
+ # load some variables from the params file
PULLCODE = False
if main.params[ 'Git' ] == 'True':
PULLCODE = True
@@ -57,38 +57,34 @@
cellName = main.params[ 'ENV' ][ 'cellName' ]
# set global variables
- global ONOS1Ip
global ONOS1Port
- global ONOS2Ip
global ONOS2Port
- global ONOS3Ip
global ONOS3Port
- global ONOS4Ip
global ONOS4Port
- global ONOS5Ip
global ONOS5Port
- global ONOS6Ip
global ONOS6Port
- global ONOS7Ip
global ONOS7Port
global numControllers
-
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
- ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
- ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
- ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
- ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
- ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
- ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
- ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
- ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
- ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
- ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
numControllers = int( main.params[ 'num_controllers' ] )
+ # FIXME: just get controller port from params?
+ # TODO: do we really need all these?
+ ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
+ ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
+ ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
+ ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
+ ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
+ ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
+ ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
+
+ global CLIs
+ CLIs = []
+ global nodes
+ nodes = []
+ for i in range( 1, numControllers + 1 ):
+ CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+ nodes.append( getattr( main, 'ONOS' + str( i ) ) )
+
main.step( "Applying cell variable to environment" )
cellResult = main.ONOSbench.setCell( cellName )
verifyResult = main.ONOSbench.verifyCell()
@@ -96,14 +92,10 @@
# FIXME:this is short term fix
main.log.report( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
+
main.log.report( "Uninstalling ONOS" )
- main.ONOSbench.onosUninstall( ONOS1Ip )
- main.ONOSbench.onosUninstall( ONOS2Ip )
- main.ONOSbench.onosUninstall( ONOS3Ip )
- main.ONOSbench.onosUninstall( ONOS4Ip )
- main.ONOSbench.onosUninstall( ONOS5Ip )
- main.ONOSbench.onosUninstall( ONOS6Ip )
- main.ONOSbench.onosUninstall( ONOS7Ip )
+ for node in nodes:
+ main.ONOSbench.onosUninstall( node.ip_address )
cleanInstallResult = main.TRUE
gitPullResult = main.TRUE
@@ -116,6 +108,8 @@
main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
+ if gitPullResult == main.ERROR:
+ main.log.error( "Error pulling git branch" )
main.step( "Using mvn clean & install" )
cleanInstallResult = main.ONOSbench.cleanInstall()
@@ -128,76 +122,38 @@
packageResult = main.ONOSbench.onosPackage()
main.step( "Installing ONOS package" )
- onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS1Ip )
- onos2InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS2Ip )
- onos3InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS3Ip )
- onos4InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS4Ip )
- onos5InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS5Ip )
- onos6InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS6Ip )
- onos7InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS7Ip )
- onosInstallResult = onos1InstallResult and onos2InstallResult\
- and onos3InstallResult and onos4InstallResult\
- and onos5InstallResult and onos6InstallResult\
- and onos7InstallResult
+ onosInstallResult = main.TRUE
+ for node in nodes:
+ tmpResult = main.ONOSbench.onosInstall( options="-f",
+ node=node.ip_address )
+ onosInstallResult = onosInstallResult and tmpResult
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
- onos1Isup = main.ONOSbench.isup( ONOS1Ip )
- if not onos1Isup:
- main.log.report( "ONOS1 didn't start!" )
- main.ONOSbench.onosStop( ONOS1Ip )
- main.ONOSbench.onosStart( ONOS1Ip )
- onos2Isup = main.ONOSbench.isup( ONOS2Ip )
- if not onos2Isup:
- main.log.report( "ONOS2 didn't start!" )
- main.ONOSbench.onosStop( ONOS2Ip )
- main.ONOSbench.onosStart( ONOS2Ip )
- onos3Isup = main.ONOSbench.isup( ONOS3Ip )
- if not onos3Isup:
- main.log.report( "ONOS3 didn't start!" )
- main.ONOSbench.onosStop( ONOS3Ip )
- main.ONOSbench.onosStart( ONOS3Ip )
- onos4Isup = main.ONOSbench.isup( ONOS4Ip )
- if not onos4Isup:
- main.log.report( "ONOS4 didn't start!" )
- main.ONOSbench.onosStop( ONOS4Ip )
- main.ONOSbench.onosStart( ONOS4Ip )
- onos5Isup = main.ONOSbench.isup( ONOS5Ip )
- if not onos5Isup:
- main.log.report( "ONOS5 didn't start!" )
- main.ONOSbench.onosStop( ONOS5Ip )
- main.ONOSbench.onosStart( ONOS5Ip )
- onos6Isup = main.ONOSbench.isup( ONOS6Ip )
- if not onos6Isup:
- main.log.report( "ONOS6 didn't start!" )
- main.ONOSbench.onosStop( ONOS6Ip )
- main.ONOSbench.onosStart( ONOS6Ip )
- onos7Isup = main.ONOSbench.isup( ONOS7Ip )
- if not onos7Isup:
- main.log.report( "ONOS7 didn't start!" )
- main.ONOSbench.onosStop( ONOS7Ip )
- main.ONOSbench.onosStart( ONOS7Ip )
- onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
- and onos4Isup and onos5Isup and onos6Isup and onos7Isup
+ onosIsupResult = main.TRUE
+ for node in nodes:
+ started = main.ONOSbench.isup( node.ip_address )
+ if not started:
+ main.log.report( node.name + " didn't start!" )
+ main.ONOSbench.onosStop( node.ip_address )
+ main.ONOSbench.onosStart( node.ip_address )
+ onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
- cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
- cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
- cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
- cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
- cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
- cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
- cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
- cliResults = cliResult1 and cliResult2 and cliResult3 and\
- cliResult4 and cliResult5 and cliResult6 and cliResult7
+ main.log.step( "Starting ONOS CLI sessions" )
+ cliResults = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].startOnosCli,
+ name="startOnosCli-" + str( i ),
+ args=[nodes[i].ip_address] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ cliResults = cliResults and t.result
main.step( "Start Packet Capture MN" )
main.Mininet2.startTcpdump(
@@ -223,22 +179,36 @@
Assign mastership to controllers
"""
import re
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert ONOS1Port, "ONOS1Port not defined"
+ assert ONOS2Port, "ONOS2Port not defined"
+ assert ONOS3Port, "ONOS3Port not defined"
+ assert ONOS4Port, "ONOS4Port not defined"
+ assert ONOS5Port, "ONOS5Port not defined"
+ assert ONOS6Port, "ONOS6Port not defined"
+ assert ONOS7Port, "ONOS7Port not defined"
main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
main.step( "Assign switches to controllers" )
+ # TODO: rewrite this function to take lists of ips and ports?
+ # or list of tuples?
for i in range( 1, 29 ):
main.Mininet1.assignSwController(
sw=str( i ),
count=numControllers,
- ip1=ONOS1Ip, port1=ONOS1Port,
- ip2=ONOS2Ip, port2=ONOS2Port,
- ip3=ONOS3Ip, port3=ONOS3Port,
- ip4=ONOS4Ip, port4=ONOS4Port,
- ip5=ONOS5Ip, port5=ONOS5Port,
- ip6=ONOS6Ip, port6=ONOS6Port,
- ip7=ONOS7Ip, port7=ONOS7Port )
+ ip1=nodes[ 0 ].ip_address, port1=ONOS1Port,
+ ip2=nodes[ 1 ].ip_address, port2=ONOS2Port,
+ ip3=nodes[ 2 ].ip_address, port3=ONOS3Port,
+ ip4=nodes[ 3 ].ip_address, port4=ONOS4Port,
+ ip5=nodes[ 4 ].ip_address, port5=ONOS5Port,
+ ip6=nodes[ 5 ].ip_address, port6=ONOS6Port,
+ ip7=nodes[ 6 ].ip_address, port7=ONOS7Port )
mastershipCheck = main.TRUE
for i in range( 1, 29 ):
@@ -247,16 +217,11 @@
main.log.info( str( response ) )
except Exception:
main.log.info( repr( response ) )
- if re.search( "tcp:" + ONOS1Ip, response )\
- and re.search( "tcp:" + ONOS2Ip, response )\
- and re.search( "tcp:" + ONOS3Ip, response )\
- and re.search( "tcp:" + ONOS4Ip, response )\
- and re.search( "tcp:" + ONOS5Ip, response )\
- and re.search( "tcp:" + ONOS6Ip, response )\
- and re.search( "tcp:" + ONOS7Ip, response ):
- mastershipCheck = mastershipCheck and main.TRUE
- else:
- mastershipCheck = main.FALSE
+ for node in nodes:
+ if re.search( "tcp:" + node.ip_address, response ):
+ mastershipCheck = mastershipCheck and main.TRUE
+ else:
+ mastershipCheck = main.FALSE
if mastershipCheck == main.TRUE:
main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
@@ -264,19 +229,20 @@
actual=mastershipCheck,
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
-
+ #FIXME: turning off because of ONOS-1286
# Manually assign mastership to the controller we want
roleCall = main.TRUE
roleCheck = main.TRUE
try:
# Assign switch
+ ip = nodes[ 0 ].ip_address # ONOS1
deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
assert deviceId, "No device id for s1 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS1Ip )
+ ip )
# Check assignment
- if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
@@ -286,21 +252,22 @@
assert deviceId, "No device id for s28 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS1Ip )
+ ip )
# Check assignment
- if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 1 ].ip_address # ONOS2
# Assign switch
deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
assert deviceId, "No device id for s2 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS2Ip )
+ ip )
# Check assignment
- if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
@@ -310,21 +277,22 @@
assert deviceId, "No device id for s3 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS2Ip )
+ ip )
# Check assignment
- if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 2 ].ip_address # ONOS3
# Assign switch
deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
assert deviceId, "No device id for s5 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS3Ip )
+ ip )
# Check assignment
- if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
@@ -334,65 +302,68 @@
assert deviceId, "No device id for s6 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS3Ip )
+ ip )
# Check assignment
- if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 3 ].ip_address # ONOS4
# Assign switch
deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
assert deviceId, "No device id for s4 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS4Ip )
+ ip )
# Check assignment
- if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 4 ].ip_address # ONOS5
for i in range( 8, 18 ):
dpid = '3' + str( i ).zfill( 3 )
deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
assert deviceId, "No device id for s%i in ONOS" % i
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS5Ip )
+ ip )
# Check assignment
- if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 5 ].ip_address # ONOS6
deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
assert deviceId, "No device id for s7 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS6Ip )
+ ip )
# Check assignment
- if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 6 ].ip_address # ONOS7
for i in range( 18, 28 ):
dpid = '6' + str( i ).zfill( 3 )
deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
assert deviceId, "No device id for s%i in ONOS" % i
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS7Ip )
+ ip )
# Check assignment
- if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
except ( AttributeError, AssertionError ):
main.log.exception( "Something is wrong with ONOS device view" )
main.log.info( main.ONOScli1.devices() )
-
utilities.assert_equals(
expect=main.TRUE,
actual=roleCall,
@@ -419,6 +390,11 @@
# datastore!
import time
import json
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
main.log.report( "Adding host intents" )
main.case( "Adding host Intents" )
@@ -427,39 +403,50 @@
# install onos-app-fwd
main.log.info( "Install reactive forwarding app" )
- main.ONOScli1.featureInstall( "onos-app-fwd" )
- main.ONOScli2.featureInstall( "onos-app-fwd" )
- main.ONOScli3.featureInstall( "onos-app-fwd" )
- main.ONOScli4.featureInstall( "onos-app-fwd" )
- main.ONOScli5.featureInstall( "onos-app-fwd" )
- main.ONOScli6.featureInstall( "onos-app-fwd" )
- main.ONOScli7.featureInstall( "onos-app-fwd" )
+ appResults = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].featureInstall,
+ name="featureInstall-" + str( i ),
+ args=["onos-app-fwd"] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ appResults = appResults and t.result
# REACTIVE FWD test
pingResult = main.FALSE
- time1 = time.time()
- pingResult = main.Mininet1.pingall()
- utilities.assert_equals(
- expect=main.TRUE,
- actual=pingResult,
- onpass="Reactive Pingall test passed",
- onfail="Reactive Pingall failed, one or more ping pairs failed" )
- time2 = time.time()
- main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
+ for i in range(2): # Retry if pingall fails first time
+ time1 = time.time()
+ pingResult = main.Mininet1.pingall()
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=pingResult,
+ onpass="Reactive Pingall test passed",
+ onfail="Reactive Pingall failed, one or more ping pairs failed" )
+ time2 = time.time()
+ main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
# uninstall onos-app-fwd
main.log.info( "Uninstall reactive forwarding app" )
- main.ONOScli1.featureUninstall( "onos-app-fwd" )
- main.ONOScli2.featureUninstall( "onos-app-fwd" )
- main.ONOScli3.featureUninstall( "onos-app-fwd" )
- main.ONOScli4.featureUninstall( "onos-app-fwd" )
- main.ONOScli5.featureUninstall( "onos-app-fwd" )
- main.ONOScli6.featureUninstall( "onos-app-fwd" )
- main.ONOScli7.featureUninstall( "onos-app-fwd" )
- # timeout for fwd flows
- time.sleep( 10 )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].featureUninstall,
+ name="featureUninstall-" + str( i ),
+ args=["onos-app-fwd"] )
+ threads.append( t )
+ t.start()
- main.step( "Add host intents" )
+ for t in threads:
+ t.join()
+ appResults = appResults and t.result
+
+ # timeout for fwd flows
+ time.sleep( 11 )
+
+ main.step( "Add host intents" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -481,28 +468,33 @@
host1Id = host1Dict.get( 'id', None )
host2Id = host2Dict.get( 'id', None )
if host1Id and host2Id:
-
- tmpId = main.ONOScli1.addHostIntent(
- host1Id,
- host2Id )
+ nodeNum = ( i % 7 )
+ tmpId = CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
if tmpId:
main.log.info( "Added intent with id: " + tmpId )
intentIds.append( tmpId )
else:
- main.log.error( "addHostIntent reutrned None" )
+ main.log.error( "addHostIntent returned: " +
+ repr( tmpId ) )
else:
- main.log.error( "Error, getHost() failed" )
- main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.error( "Error, getHost() failed for h" + str( i ) +
+ " and/or h" + str( i + 10 ) )
+ hosts = CLIs[ 0 ].hosts()
+ main.log.warn( "Hosts output: " )
+ try:
+ main.log.warn( json.dumps( json.loads( hosts ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( hosts ) )
hostResult = main.FALSE
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
main.log.info( "Intents in ONOS: " + str( onosIds ) )
for intent in intentIds:
if intent in onosIds:
- pass # intent submitted is still in onos
+ pass # intent submitted is in onos
else:
intentAddResult = False
# Print the intent states
@@ -511,12 +503,15 @@
installedCheck = True
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
- for intent in json.loads( intents ): # Iter through intents of a node
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents" )
# add submitted intents not in the store
tmplist = [ i for i, s in intentStates ]
missingIntents = False
@@ -529,17 +524,62 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- # for node in nodes:
- # node.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
pendingMap = main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
+
intentAddResult = bool( pingResult and hostResult and intentAddResult
and not missingIntents and installedCheck )
utilities.assert_equals(
@@ -561,13 +601,16 @@
intentStates = []
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
- for intent in json.loads( intents ):
- # Iter through intents of a node
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ # Iter through intents of a node
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents" )
# add submitted intents not in the store
tmplist = [ i for i, s in intentStates ]
for i in intentIds:
@@ -578,20 +621,73 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
def CASE4( self, main ):
"""
Ping across added host intents
"""
import json
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
description = " Ping across added host intents"
main.log.report( description )
main.case( description )
@@ -610,11 +706,15 @@
main.log.report(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
- main.log.warn( "ONSO1 intents: " )
- main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
if PingResult == main.TRUE:
main.log.report(
"Intents have been installed correctly and verified by pings" )
@@ -632,28 +732,79 @@
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
# Iter through intents of a node
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
+
if not installedCheck:
- main.log.info( "Waiting 60 seconds to see if intent states change" )
+ main.log.info( "Waiting 60 seconds to see if the state of " +
+ "intents change" )
time.sleep( 60 )
# Print the intent states
intents = main.ONOScli1.intents()
@@ -661,32 +812,87 @@
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
# Iter through intents of a node
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
def CASE5( self, main ):
"""
Reading state of ONOS
"""
import json
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
@@ -694,344 +900,231 @@
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
- # We can then compare them with eachother and also with past states
+ # We can then compare them with each other and also with past states
- main.step( "Get the Mastership of each switch from each controller" )
+ main.step( "Check that each switch has a master" )
global mastershipState
- mastershipState = []
+ mastershipState = '[]'
# Assert that each device has a master
- ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
- ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
- ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
- ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
- ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
- ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
- ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
- rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
- ONOS3MasterNotNull and ONOS4MasterNotNull and\
- ONOS5MasterNotNull and ONOS6MasterNotNull and\
- ONOS7MasterNotNull
+ rolesNotNull = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].rolesNotNull,
+ name="rolesNotNull-" + str( i ),
+ args=[] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ rolesNotNull = rolesNotNull and t.result
utilities.assert_equals(
expect=main.TRUE,
actual=rolesNotNull,
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
- ONOS1Mastership = main.ONOScli1.roles()
- ONOS2Mastership = main.ONOScli2.roles()
- ONOS3Mastership = main.ONOScli3.roles()
- ONOS4Mastership = main.ONOScli4.roles()
- ONOS5Mastership = main.ONOScli5.roles()
- ONOS6Mastership = main.ONOScli6.roles()
- ONOS7Mastership = main.ONOScli7.roles()
- if "Error" in ONOS1Mastership or not ONOS1Mastership\
- or "Error" in ONOS2Mastership or not ONOS2Mastership\
- or "Error" in ONOS3Mastership or not ONOS3Mastership\
- or "Error" in ONOS4Mastership or not ONOS4Mastership\
- or "Error" in ONOS5Mastership or not ONOS5Mastership\
- or "Error" in ONOS6Mastership or not ONOS6Mastership\
- or "Error" in ONOS7Mastership or not ONOS7Mastership:
- main.log.report( "Error in getting ONOS roles" )
- main.log.warn(
- "ONOS1 mastership response: " +
- repr( ONOS1Mastership ) )
- main.log.warn(
- "ONOS2 mastership response: " +
- repr( ONOS2Mastership ) )
- main.log.warn(
- "ONOS3 mastership response: " +
- repr( ONOS3Mastership ) )
- main.log.warn(
- "ONOS4 mastership response: " +
- repr( ONOS4Mastership ) )
- main.log.warn(
- "ONOS5 mastership response: " +
- repr( ONOS5Mastership ) )
- main.log.warn(
- "ONOS6 mastership response: " +
- repr( ONOS6Mastership ) )
- main.log.warn(
- "ONOS7 mastership response: " +
- repr( ONOS7Mastership ) )
- consistentMastership = main.FALSE
- elif ONOS1Mastership == ONOS2Mastership\
- and ONOS1Mastership == ONOS3Mastership\
- and ONOS1Mastership == ONOS4Mastership\
- and ONOS1Mastership == ONOS5Mastership\
- and ONOS1Mastership == ONOS6Mastership\
- and ONOS1Mastership == ONOS7Mastership:
- mastershipState = ONOS1Mastership
- consistentMastership = main.TRUE
+ main.step( "Get the Mastership of each switch from each controller" )
+ ONOSMastership = []
+ mastershipCheck = main.FALSE
+ consistentMastership = True
+ rolesResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].roles,
+ name="roles-" + str( i ),
+ args=[] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSMastership.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " roles" )
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " mastership response: " +
+ repr( ONOSMastership[i] ) )
+ rolesResults = False
+ utilities.assert_equals(
+ expect=True,
+ actual=rolesResults,
+ onpass="No error in reading roles output",
+ onfail="Error in reading roles from ONOS" )
+
+ main.step( "Check for consistency in roles from each controller" )
+ if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
main.log.report(
"Switch roles are consistent across all ONOS nodes" )
else:
- main.log.warn(
- "ONOS1 roles: ",
- json.dumps(
- json.loads( ONOS1Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS2 roles: ",
- json.dumps(
- json.loads( ONOS2Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS3 roles: ",
- json.dumps(
- json.loads( ONOS3Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS4 roles: ",
- json.dumps(
- json.loads( ONOS4Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS5 roles: ",
- json.dumps(
- json.loads( ONOS5Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS6 roles: ",
- json.dumps(
- json.loads( ONOS6Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS7 roles: ",
- json.dumps(
- json.loads( ONOS7Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- consistentMastership = main.FALSE
+ consistentMastership = False
utilities.assert_equals(
- expect=main.TRUE,
+ expect=True,
actual=consistentMastership,
onpass="Switch roles are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of switch roles" )
+ if rolesResults and not consistentMastership:
+ for i in range( numControllers ):
+ try:
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " roles: ",
+ json.dumps(
+ json.loads( ONOSMastership[ i ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( ONOSMastership[ i ] ) )
+ elif rolesResults and consistentMastership:
+ mastershipCheck = main.TRUE
+ mastershipState = ONOSMastership[ 0 ]
+
main.step( "Get the intents from each controller" )
global intentState
intentState = []
- ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
- ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
- ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
- ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
- ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
- ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
- ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
+ ONOSIntents = []
intentCheck = main.FALSE
- if "Error" in ONOS1Intents or not ONOS1Intents\
- or "Error" in ONOS2Intents or not ONOS2Intents\
- or "Error" in ONOS3Intents or not ONOS3Intents\
- or "Error" in ONOS4Intents or not ONOS4Intents\
- or "Error" in ONOS5Intents or not ONOS5Intents\
- or "Error" in ONOS6Intents or not ONOS6Intents\
- or "Error" in ONOS7Intents or not ONOS7Intents:
- main.log.report( "Error in getting ONOS intents" )
- main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
- main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
- main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
- main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
- main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
- main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
- main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
- elif ONOS1Intents == ONOS2Intents\
- and ONOS1Intents == ONOS3Intents\
- and ONOS1Intents == ONOS4Intents\
- and ONOS1Intents == ONOS5Intents\
- and ONOS1Intents == ONOS6Intents\
- and ONOS1Intents == ONOS7Intents:
- intentState = ONOS1Intents
- intentCheck = main.TRUE
- main.log.report( "Intents are consistent across all ONOS nodes" )
- else:
- main.log.warn(
- "ONOS1 intents: ",
- json.dumps(
- json.loads( ONOS1Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS2 intents: ",
- json.dumps(
- json.loads( ONOS2Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS3 intents: ",
- json.dumps(
- json.loads( ONOS3Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS4 intents: ",
- json.dumps(
- json.loads( ONOS4Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS5 intents: ",
- json.dumps(
- json.loads( ONOS5Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS6 intents: ",
- json.dumps(
- json.loads( ONOS6Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS7 intents: ",
- json.dumps(
- json.loads( ONOS7Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
+ consistentIntents = True
+ intentsResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].intents,
+ name="intents-" + str( i ),
+ args=[],
+ kwargs={ 'jsonFormat': True } )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSIntents.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " intents" )
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
+ repr( ONOSIntents[ i ] ) )
+ intentsResults = False
utilities.assert_equals(
- expect=main.TRUE,
- actual=intentCheck,
+ expect=True,
+ actual=intentsResults,
+ onpass="No error in reading intents output",
+ onfail="Error in reading intents from ONOS" )
+
+ main.step( "Check for consistency in Intents from each controller" )
+ if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
+ main.log.report( "Intents are consistent across all ONOS " +
+ "nodes" )
+ else:
+ consistentIntents = False
+ main.log.report( "Intents not consistent" )
+ utilities.assert_equals(
+ expect=True,
+ actual=consistentIntents,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ if intentsResults and not consistentIntents:
+ n = len(ONOSIntents)
+ main.log.warn( "ONOS" + str( n ) + " intents: " )
+ main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ for i in range( numControllers ):
+ if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
+ main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ else:
+ main.log.warn( nodes[ i ].name + " intents match ONOS" +
+ str( n ) + " intents" )
+ elif intentsResults and consistentIntents:
+ intentCheck = main.TRUE
+ intentState = ONOSIntents[ 0 ]
+
main.step( "Get the flows from each controller" )
global flowState
flowState = []
+ ONOSFlows = []
+ ONOSFlowsJson = []
flowCheck = main.FALSE
- try:
- ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
- ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
- ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
- ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
- ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
- ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
- ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
- assert ONOS1Flows, "ONOS1 Flows should not be empty"
- assert ONOS2Flows, "ONOS2 Flows should not be empty"
- assert ONOS3Flows, "ONOS3 Flows should not be empty"
- assert ONOS4Flows, "ONOS4 Flows should not be empty"
- assert ONOS5Flows, "ONOS5 Flows should not be empty"
- assert ONOS6Flows, "ONOS6 Flows should not be empty"
- assert ONOS7Flows, "ONOS7 Flows should not be empty"
- assert "Error" not in ONOS1Flows, "ONOS1 Flows contains 'Error'"
- assert "Error" not in ONOS2Flows, "ONOS2 Flows contains 'Error'"
- assert "Error" not in ONOS3Flows, "ONOS3 Flows contains 'Error'"
- assert "Error" not in ONOS4Flows, "ONOS4 Flows contains 'Error'"
- assert "Error" not in ONOS5Flows, "ONOS5 Flows contains 'Error'"
- assert "Error" not in ONOS6Flows, "ONOS6 Flows contains 'Error'"
- assert "Error" not in ONOS7Flows, "ONOS7 Flows contains 'Error'"
- ONOS1FlowsJson = json.loads( ONOS1Flows )
- ONOS2FlowsJson = json.loads( ONOS2Flows )
- ONOS3FlowsJson = json.loads( ONOS3Flows )
- ONOS4FlowsJson = json.loads( ONOS4Flows )
- ONOS5FlowsJson = json.loads( ONOS5Flows )
- ONOS6FlowsJson = json.loads( ONOS6Flows )
- ONOS7FlowsJson = json.loads( ONOS7Flows )
- except ( ValueError, AssertionError ): # From json.loads, or asserts
- main.log.exception( "One or more 'flows' responses from " +
- "ONOS couldn't be decoded." )
- main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
- main.log.warn( "ONOS2 flows repsponse: " + ONOS2Flows )
- main.log.warn( "ONOS3 flows repsponse: " + ONOS3Flows )
- main.log.warn( "ONOS4 flows repsponse: " + ONOS4Flows )
- main.log.warn( "ONOS5 flows repsponse: " + ONOS5Flows )
- main.log.warn( "ONOS6 flows repsponse: " + ONOS6Flows )
- main.log.warn( "ONOS7 flows repsponse: " + ONOS7Flows )
- else: # No exceptions
- if len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
- # TODO: Do a better check, maybe compare flows on switches?
- # NOTE Possible issue with this not always being set?
- flowState = ONOS1Flows
- flowCheck = main.TRUE
- main.log.report( "Flow count is consistent across all" +
- " ONOS nodes" )
+ consistentFlows = True
+ flowsResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].flows,
+ name="flows-" + str( i ),
+ args=[],
+ kwargs={ 'jsonFormat': True } )
+ threads.append( t )
+ t.start()
+
+ time.sleep(30)
+ for t in threads:
+ t.join()
+ result = t.result
+ ONOSFlows.append( result )
+
+ for i in range( numControllers ):
+ num = str( i + 1 )
+ if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
+ main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.warn( "ONOS" + num + " flows response: " +
+ repr( ONOSFlows[ i ] ) )
+ flowsResults = False
+ ONOSFlowsJson.append( None )
else:
- main.log.warn( "ONOS1 flows: " +
- json.dumps( ONOS1FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS2 flows: " +
- json.dumps( ONOS2FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS3 flows: " +
- json.dumps( ONOS3FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS4 flows: " +
- json.dumps( ONOS4FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS5 flows: " +
- json.dumps( ONOS5FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS6 flows: " +
- json.dumps( ONOS6FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS7 flows: " +
- json.dumps( ONOS7FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ try:
+ ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
+ except ( ValueError, TypeError ):
+ # FIXME: change this to log.error?
+ main.log.exception( "Error in parsing ONOS" + num +
+ " response as json." )
+ main.log.error( repr( ONOSFlows[ i ] ) )
+ ONOSFlowsJson.append( None )
+ flowsResults = False
utilities.assert_equals(
- expect=main.TRUE,
- actual=flowCheck,
+ expect=True,
+ actual=flowsResults,
+ onpass="No error in reading flows output",
+ onfail="Error in reading flows from ONOS" )
+
+ main.step( "Check for consistency in Flows from each controller" )
+ tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
+ if all( tmp ):
+ main.log.report( "Flow count is consistent across all ONOS nodes" )
+ else:
+ consistentFlows = False
+ utilities.assert_equals(
+ expect=True,
+ actual=consistentFlows,
onpass="The flow count is consistent across all ONOS nodes",
onfail="ONOS nodes have different flow counts" )
+ if flowsResults and not consistentFlows:
+ for i in range( numControllers ):
+ try:
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " flows: " +
+ json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
+ indent=4, separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " flows: " +
+ repr( ONOSFlows[ i ] ) )
+ elif flowsResults and consistentFlows:
+ flowCheck = main.TRUE
+ flowState = ONOSFlows[ 0 ]
+
main.step( "Get the OF Table entries" )
global flows
flows = []
@@ -1086,64 +1179,80 @@
main.step( "Create TestONTopology object" )
ctrls = []
- count = 1
- while True:
- temp = ()
- if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
- temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
- temp = temp + ( "ONOS" + str( count ), )
- temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
- temp = temp + \
- ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
- ctrls.append( temp )
- count = count + 1
- else:
- break
- MNTopo = TestONTopology(
- main.Mininet1,
- ctrls ) # can also add Intent API info for intent operations
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
main.step( "Collecting topology information from ONOS" )
devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].devices,
+ name="devices-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ devices.append( t.result )
hosts = []
- hosts.append( json.loads( main.ONOScli1.hosts() ) )
- hosts.append( json.loads( main.ONOScli2.hosts() ) )
- hosts.append( json.loads( main.ONOScli3.hosts() ) )
- hosts.append( json.loads( main.ONOScli4.hosts() ) )
- hosts.append( json.loads( main.ONOScli5.hosts() ) )
- hosts.append( json.loads( main.ONOScli6.hosts() ) )
- hosts.append( json.loads( main.ONOScli7.hosts() ) )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].hosts,
+ name="hosts-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ try:
+ hosts.append( json.loads( t.result ) )
+ except ( ValueError, TypeError ):
+ # FIXME: better handling of this, print which node
+ # Maybe use thread name?
+ main.log.exception( "Error parsing json output of hosts" )
+ # FIXME: should this be an empty json object instead?
+ hosts.append( None )
+
ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].ports,
+ name="ports-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ports.append( t.result )
links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].links,
+ name="links-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ links.append( t.result )
clusters = []
- clusters.append( main.ONOScli1.clusters() )
- clusters.append( main.ONOScli2.clusters() )
- clusters.append( main.ONOScli3.clusters() )
- clusters.append( main.ONOScli4.clusters() )
- clusters.append( main.ONOScli5.clusters() )
- clusters.append( main.ONOScli6.clusters() )
- clusters.append( main.ONOScli7.clusters() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].clusters,
+ name="clusters-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ clusters.append( t.result )
# Compare json objects for hosts and dataplane clusters
# hosts
@@ -1177,10 +1286,9 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host.get( 'ips', [] ) == []:
- main.log.error(
- "DEBUG:Error with host ips on controller" +
- controllerStr + ": " + str( host ) )
+ if not host.get( 'ips', [ ] ):
+ main.log.error( "DEBUG:Error with host ips on controller" +
+ controllerStr + ": " + str( host ) )
ipResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
@@ -1191,12 +1299,12 @@
# Strongly connected clusters of devices
consistentClustersResult = main.TRUE
for controller in range( len( clusters ) ):
+ controllerStr = str( controller + 1 )
if "Error" not in clusters[ controller ]:
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
- controllerStr +
+ main.log.report( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
@@ -1213,7 +1321,11 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
- numClusters = len( json.loads( clusters[ 0 ] ) )
+ try:
+ numClusters = len( json.loads( clusters[ 0 ] ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing clusters[0]: " +
+ repr( clusters[ 0 ] ) )
clusterResults = main.FALSE
if numClusters == 1:
clusterResults = main.TRUE
@@ -1232,8 +1344,7 @@
if devices[ controller ] or "Error" not in devices[ controller ]:
currentDevicesResult = main.Mininet1.compareSwitches(
MNTopo,
- json.loads(
- devices[ controller ] ) )
+ json.loads( devices[ controller ] ) )
else:
currentDevicesResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
@@ -1246,8 +1357,7 @@
if ports[ controller ] or "Error" not in ports[ controller ]:
currentPortsResult = main.Mininet1.comparePorts(
MNTopo,
- json.loads(
- ports[ controller ] ) )
+ json.loads( ports[ controller ] ) )
else:
currentPortsResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
@@ -1260,8 +1370,7 @@
if links[ controller ] or "Error" not in links[ controller ]:
currentLinksResult = main.Mininet1.compareLinks(
MNTopo,
- json.loads(
- links[ controller ] ) )
+ json.loads( links[ controller ] ) )
else:
currentLinksResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
@@ -1275,16 +1384,17 @@
portsResults = portsResults and currentPortsResult
linksResults = linksResults and currentLinksResult
- topoResult = devicesResults and portsResults and linksResults\
- and consistentHostsResult and consistentClustersResult\
- and clusterResults and ipResult
+ topoResult = ( devicesResults and portsResults and linksResults
+ and consistentHostsResult and consistentClustersResult
+ and clusterResults and ipResult )
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
finalAssert = main.TRUE
- finalAssert = finalAssert and topoResult and flowCheck \
- and intentCheck and consistentMastership and rolesNotNull
+ finalAssert = ( finalAssert and topoResult and flowCheck
+ and intentCheck and consistentMastership
+ and rolesNotNull )
utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
onpass="State check successful",
onfail="State check NOT successful" )
@@ -1293,42 +1403,43 @@
"""
The Failure case.
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
main.log.report( "Restart entire ONOS cluster" )
- main.log.case( "Restart entire ONOS cluster" )
- main.ONOSbench.onosKill( ONOS1Ip )
- main.ONOSbench.onosKill( ONOS2Ip )
- main.ONOSbench.onosKill( ONOS3Ip )
- main.ONOSbench.onosKill( ONOS4Ip )
- main.ONOSbench.onosKill( ONOS5Ip )
- main.ONOSbench.onosKill( ONOS6Ip )
- main.ONOSbench.onosKill( ONOS7Ip )
+ main.case( "Restart entire ONOS cluster" )
+ main.step( "Killing ONOS nodes" )
+ killResults = main.TRUE
+ for node in nodes:
+ killed = main.ONOSbench.onosKill( node.ip_address )
+ killResults = killResults and killed
main.step( "Checking if ONOS is up yet" )
- count = 0
- onosIsupResult = main.FALSE
- while onosIsupResult == main.FALSE and count < 10:
- onos1Isup = main.ONOSbench.isup( ONOS1Ip )
- onos2Isup = main.ONOSbench.isup( ONOS2Ip )
- onos3Isup = main.ONOSbench.isup( ONOS3Ip )
- onos4Isup = main.ONOSbench.isup( ONOS4Ip )
- onos5Isup = main.ONOSbench.isup( ONOS5Ip )
- onos6Isup = main.ONOSbench.isup( ONOS6Ip )
- onos7Isup = main.ONOSbench.isup( ONOS7Ip )
- onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
- and onos4Isup and onos5Isup and onos6Isup and onos7Isup
- count = count + 1
- # TODO: if it becomes an issue, we can retry this step a few times
+ for i in range( 2 ):
+ onosIsupResult = main.TRUE
+ for node in nodes:
+ started = main.ONOSbench.isup( node.ip_address )
+ if not started:
+ main.log.report( node.name + " didn't start!" )
+ onosIsupResult = onosIsupResult and started
+ if onosIsupResult == main.TRUE:
+ break
- cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
- cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
- cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
- cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
- cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
- cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
- cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
- cliResults = cliResult1 and cliResult2 and cliResult3\
- and cliResult4 and cliResult5 and cliResult6\
- and cliResult7
+ main.log.step( "Starting ONOS CLI sessions" )
+ cliResults = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].startOnosCli,
+ name="startOnosCli-" + str( i ),
+ args=[nodes[i].ip_address] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ cliResults = cliResults and t.result
caseResults = main.TRUE and onosIsupResult and cliResults
utilities.assert_equals( expect=main.TRUE, actual=caseResults,
@@ -1340,107 +1451,103 @@
Check state after ONOS failure
"""
import json
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
main.case( "Running ONOS Constant State Tests" )
+ main.step( "Check that each switch has a master" )
# Assert that each device has a master
- ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
- ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
- ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
- ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
- ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
- ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
- ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
- rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
- ONOS3MasterNotNull and ONOS4MasterNotNull and\
- ONOS5MasterNotNull and ONOS6MasterNotNull and\
- ONOS7MasterNotNull
+ rolesNotNull = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].rolesNotNull,
+ name="rolesNotNull-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ rolesNotNull = rolesNotNull and t.result
utilities.assert_equals(
expect=main.TRUE,
actual=rolesNotNull,
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
- main.step( "Check if switch roles are consistent across all nodes" )
- ONOS1Mastership = main.ONOScli1.roles()
- ONOS2Mastership = main.ONOScli2.roles()
- ONOS3Mastership = main.ONOScli3.roles()
- ONOS4Mastership = main.ONOScli4.roles()
- ONOS5Mastership = main.ONOScli5.roles()
- ONOS6Mastership = main.ONOScli6.roles()
- ONOS7Mastership = main.ONOScli7.roles()
- if "Error" in ONOS1Mastership or not ONOS1Mastership\
- or "Error" in ONOS2Mastership or not ONOS2Mastership\
- or "Error" in ONOS3Mastership or not ONOS3Mastership\
- or "Error" in ONOS4Mastership or not ONOS4Mastership\
- or "Error" in ONOS5Mastership or not ONOS5Mastership\
- or "Error" in ONOS6Mastership or not ONOS6Mastership\
- or "Error" in ONOS7Mastership or not ONOS7Mastership:
- main.log.error( "Error in getting ONOS mastership" )
- main.log.warn( "ONOS1 mastership response: " +
- repr( ONOS1Mastership ) )
- main.log.warn( "ONOS2 mastership response: " +
- repr( ONOS2Mastership ) )
- main.log.warn( "ONOS3 mastership response: " +
- repr( ONOS3Mastership ) )
- main.log.warn( "ONOS4 mastership response: " +
- repr( ONOS4Mastership ) )
- main.log.warn( "ONOS5 mastership response: " +
- repr( ONOS5Mastership ) )
- main.log.warn( "ONOS6 mastership response: " +
- repr( ONOS6Mastership ) )
- main.log.warn( "ONOS7 mastership response: " +
- repr( ONOS7Mastership ) )
- consistentMastership = main.FALSE
- elif ONOS1Mastership == ONOS2Mastership\
- and ONOS1Mastership == ONOS3Mastership\
- and ONOS1Mastership == ONOS4Mastership\
- and ONOS1Mastership == ONOS5Mastership\
- and ONOS1Mastership == ONOS6Mastership\
- and ONOS1Mastership == ONOS7Mastership:
- consistentMastership = main.TRUE
+ ONOSMastership = []
+ mastershipCheck = main.FALSE
+ consistentMastership = True
+ rolesResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].roles,
+ name="roles-" + str( i ),
+ args=[] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSMastership.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " roles" )
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " mastership response: " +
+ repr( ONOSMastership[i] ) )
+ rolesResults = False
+ utilities.assert_equals(
+ expect=True,
+ actual=rolesResults,
+ onpass="No error in reading roles output",
+ onfail="Error in reading roles from ONOS" )
+
+ main.step( "Check for consistency in roles from each controller" )
+ if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
main.log.report(
"Switch roles are consistent across all ONOS nodes" )
else:
- main.log.warn( "ONOS1 roles: ", json.dumps(
- json.loads( ONOS1Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS2 roles: ", json.dumps(
- json.loads( ONOS2Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS3 roles: ", json.dumps(
- json.loads( ONOS3Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS4 roles: ", json.dumps(
- json.loads( ONOS4Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS5 roles: ", json.dumps(
- json.loads( ONOS5Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS6 roles: ", json.dumps(
- json.loads( ONOS6Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS7 roles: ", json.dumps(
- json.loads( ONOS7Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- consistentMastership = main.FALSE
+ consistentMastership = False
utilities.assert_equals(
- expect=main.TRUE,
+ expect=True,
actual=consistentMastership,
onpass="Switch roles are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of switch roles" )
+ if rolesResults and not consistentMastership:
+ for i in range( numControllers ):
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " roles: ",
+ json.dumps(
+ json.loads( ONOSMastership[ i ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ elif rolesResults and not consistentMastership:
+ mastershipCheck = main.TRUE
+
description2 = "Compare switch roles from before failure"
main.step( description2 )
-
- currentJson = json.loads( ONOS1Mastership )
- oldJson = json.loads( mastershipState )
+ try:
+ currentJson = json.loads( ONOSMastership[0] )
+ oldJson = json.loads( mastershipState )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Something is wrong with parsing " +
+ "ONOSMastership[0] or mastershipState" )
+ main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
+ main.log.error( "mastershipState" + repr( mastershipState ) )
+ main.cleanup()
+ main.exit()
mastershipCheck = main.TRUE
for i in range( 1, 29 ):
switchDPID = str(
- main.Mininet1.getSwitchDPID(
- switch="s" +
- str( i ) ) )
-
+ main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
current = [ switch[ 'master' ] for switch in currentJson
if switchDPID in switch[ 'id' ] ]
old = [ switch[ 'master' ] for switch in oldJson
@@ -1461,83 +1568,72 @@
mastershipCheck = mastershipCheck and consistentMastership
main.step( "Get the intents and compare across all nodes" )
- ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
- ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
- ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
- ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
- ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
- ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
- ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
+ ONOSIntents = []
intentCheck = main.FALSE
- if "Error" in ONOS1Intents or not ONOS1Intents\
- or "Error" in ONOS2Intents or not ONOS2Intents\
- or "Error" in ONOS3Intents or not ONOS3Intents\
- or "Error" in ONOS4Intents or not ONOS4Intents\
- or "Error" in ONOS5Intents or not ONOS5Intents\
- or "Error" in ONOS6Intents or not ONOS6Intents\
- or "Error" in ONOS7Intents or not ONOS7Intents:
- main.log.report( "Error in getting ONOS intents" )
- main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
- main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
- main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
- main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
- main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
- main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
- main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
- elif ONOS1Intents == ONOS2Intents\
- and ONOS1Intents == ONOS3Intents\
- and ONOS1Intents == ONOS4Intents\
- and ONOS1Intents == ONOS5Intents\
- and ONOS1Intents == ONOS6Intents\
- and ONOS1Intents == ONOS7Intents:
- intentCheck = main.TRUE
- main.log.report( "Intents are consistent across all ONOS nodes" )
- else:
- main.log.warn( "ONOS1 intents: " )
- print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS2 intents: " )
- print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS3 intents: " )
- print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS4 intents: " )
- print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS5 intents: " )
- print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS6 intents: " )
- print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS7 intents: " )
- print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
+ consistentIntents = True
+ intentsResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].intents,
+ name="intents-" + str( i ),
+ args=[],
+ kwargs={ 'jsonFormat': True } )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSIntents.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " intents" )
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
+ repr( ONOSIntents[ i ] ) )
+ intentsResults = False
utilities.assert_equals(
- expect=main.TRUE,
- actual=intentCheck,
+ expect=True,
+ actual=intentsResults,
+ onpass="No error in reading intents output",
+ onfail="Error in reading intents from ONOS" )
+
+ main.step( "Check for consistency in Intents from each controller" )
+ if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
+ main.log.report( "Intents are consistent across all ONOS " +
+ "nodes" )
+ else:
+ consistentIntents = False
+ utilities.assert_equals(
+ expect=True,
+ actual=consistentIntents,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
- # Print the intent states
- intents = []
- intents.append( ONOS1Intents )
- intents.append( ONOS2Intents )
- intents.append( ONOS3Intents )
- intents.append( ONOS4Intents )
- intents.append( ONOS5Intents )
- intents.append( ONOS6Intents )
- intents.append( ONOS7Intents )
intentStates = []
- for node in intents: # Iter through ONOS nodes
+ for node in ONOSIntents: # Iter through ONOS nodes
nodeStates = []
# Iter through intents of a node
- for intent in json.loads( node ):
- nodeStates.append( intent[ 'state' ] )
+ try:
+ for intent in json.loads( node ):
+ nodeStates.append( intent[ 'state' ] )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error in parsing intents" )
+ main.log.error( repr( node ) )
intentStates.append( nodeStates )
out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
main.log.info( dict( out ) )
+ if intentsResults and not consistentIntents:
+ for i in range( numControllers ):
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
+ main.log.warn( json.dumps(
+ json.loads( ONOSIntents[ i ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ elif intentsResults and consistentIntents:
+ intentCheck = main.TRUE
+
# NOTE: Store has no durability, so intents are lost across system
# restarts
"""
@@ -1545,19 +1641,20 @@
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
sameIntents = main.TRUE
- if intentState and intentState == ONOS1Intents:
+ if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.report( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
- # what the aceptable states are
+ # what the acceptable states are
else:
try:
- main.log.warn( "ONOS1 intents: " )
- print json.dumps( json.loads( ONOS1Intents ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) )
- except Exception:
- pass
+ main.log.warn( "ONOS intents: " )
+ main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.warn( repr( ONOSIntents[0] ) )
sameIntents = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
@@ -1591,12 +1688,8 @@
main.step( "Check the continuous pings to ensure that no packets " +
"were dropped during component failure" )
- # FIXME: This check is always failing. Investigate cause
- # NOTE: this may be something to do with file permsissions
- # or slight change in format
- main.Mininet2.pingKill(
- main.params[ 'TESTONUSER' ],
- main.params[ 'TESTONIP' ] )
+ main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
+ main.params[ 'TESTONIP' ] )
LossInPings = main.FALSE
# NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range( 8, 18 ):
@@ -1625,10 +1718,8 @@
# Test of LeadershipElection
leaderList = []
leaderResult = main.TRUE
- for controller in range( 1, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- leaderN = node.electionTestLeader()
+ for cli in CLIs:
+ leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
@@ -1637,7 +1728,7 @@
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.report( "ONOS" + str( controller ) +
+ main.log.report( cli.name +
" shows no leader for the election-app was" +
" elected after the old one died" )
leaderResult = main.FALSE
@@ -1676,28 +1767,21 @@
from sts.topology.teston_topology import TestONTopology
import json
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
description = "Compare ONOS Topology view to Mininet topology"
main.case( description )
main.log.report( description )
main.step( "Create TestONTopology object" )
ctrls = []
- count = 1
- while True:
- temp = ()
- if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
- temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
- temp = temp + ( "ONOS" + str( count ), )
- temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
- temp = temp + \
- ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
- ctrls.append( temp )
- count = count + 1
- else:
- break
- MNTopo = TestONTopology(
- main.Mininet1,
- ctrls ) # can also add Intent API info for intent operations
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1711,28 +1795,40 @@
startTime = time.time()
# Give time for Gossip to work
while topoResult == main.FALSE and elapsed < 60:
- count = count + 1
+ count += 1
if count > 1:
- # TODO: Depricate STS usage
+ # TODO: Deprecate STS usage
MNTopo = TestONTopology( main.Mininet1, ctrls )
cliStart = time.time()
devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].devices,
+ name="devices-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ devices.append( t.result )
hosts = []
- hosts.append( json.loads( main.ONOScli1.hosts() ) )
- hosts.append( json.loads( main.ONOScli2.hosts() ) )
- hosts.append( json.loads( main.ONOScli3.hosts() ) )
- hosts.append( json.loads( main.ONOScli4.hosts() ) )
- hosts.append( json.loads( main.ONOScli5.hosts() ) )
- hosts.append( json.loads( main.ONOScli6.hosts() ) )
- hosts.append( json.loads( main.ONOScli7.hosts() ) )
ipResult = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].hosts,
+ name="hosts-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ try:
+ hosts.append( json.loads( t.result ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing hosts results" )
+ main.log.error( repr( t.result ) )
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
@@ -1742,29 +1838,41 @@
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].ports,
+ name="ports-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ports.append( t.result )
links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].links,
+ name="links-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ links.append( t.result )
clusters = []
- clusters.append( main.ONOScli1.clusters() )
- clusters.append( main.ONOScli2.clusters() )
- clusters.append( main.ONOScli3.clusters() )
- clusters.append( main.ONOScli4.clusters() )
- clusters.append( main.ONOScli5.clusters() )
- clusters.append( main.ONOScli6.clusters() )
- clusters.append( main.ONOScli7.clusters() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].clusters,
+ name="clusters-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ clusters.append( t.result )
elapsed = time.time() - startTime
cliTime = time.time() - cliStart
@@ -1883,7 +1991,11 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
- numClusters = len( json.loads( clusters[ 0 ] ) )
+ try:
+ numClusters = len( json.loads( clusters[ 0 ] ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing clusters[0]: " +
+ repr( clusters[0] ) )
clusterResults = main.FALSE
if numClusters == 1:
clusterResults = main.TRUE
@@ -1917,6 +2029,11 @@
Link s3-s28 down
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# NOTE: You should probably run a topology check after this
linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
@@ -1932,7 +2049,7 @@
" seconds for link down to be discovered" )
time.sleep( linkSleep )
utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
- onpass="Link down succesful",
+ onpass="Link down successful",
onfail="Failed to bring link down" )
# TODO do some sort of check here
@@ -1941,6 +2058,11 @@
Link s3-s28 up
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# NOTE: You should probably run a topology check after this
linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
@@ -1956,7 +2078,7 @@
" seconds for link up to be discovered" )
time.sleep( linkSleep )
utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
- onpass="Link up succesful",
+ onpass="Link up successful",
onfail="Failed to bring link up" )
# TODO do some sort of check here
@@ -1966,6 +2088,11 @@
"""
# NOTE: You should probably run a topology check after this
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
@@ -1989,7 +2116,7 @@
if device and device[ 'available' ] is False:
result = main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Kill switch succesful",
+ onpass="Kill switch successful",
onfail="Failed to kill switch?" )
def CASE12( self, main ):
@@ -1998,6 +2125,18 @@
"""
# NOTE: You should probably run a topology check after this
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert ONOS1Port, "ONOS1Port not defined"
+ assert ONOS2Port, "ONOS2Port not defined"
+ assert ONOS3Port, "ONOS3Port not defined"
+ assert ONOS4Port, "ONOS4Port not defined"
+ assert ONOS5Port, "ONOS5Port not defined"
+ assert ONOS6Port, "ONOS6Port not defined"
+ assert ONOS7Port, "ONOS7Port not defined"
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
switch = main.params[ 'kill' ][ 'switch' ]
@@ -2010,24 +2149,23 @@
main.step( "Add back " + switch )
main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
- # TODO: New dpid or same? Ask Thomas?
for peer in links:
main.Mininet1.addLink( switch, peer )
main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
count=numControllers,
- ip1=ONOS1Ip,
+ ip1=nodes[ 0 ].ip_address,
port1=ONOS1Port,
- ip2=ONOS2Ip,
+ ip2=nodes[ 1 ].ip_address,
port2=ONOS2Port,
- ip3=ONOS3Ip,
+ ip3=nodes[ 2 ].ip_address,
port3=ONOS3Port,
- ip4=ONOS4Ip,
+ ip4=nodes[ 3 ].ip_address,
port4=ONOS4Port,
- ip5=ONOS5Ip,
+ ip5=nodes[ 4 ].ip_address,
port5=ONOS5Port,
- ip6=ONOS6Ip,
+ ip6=nodes[ 5 ].ip_address,
port6=ONOS6Port,
- ip7=ONOS7Ip,
+ ip7=nodes[ 6 ].ip_address,
port7=ONOS7Port )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch up to be discovered" )
@@ -2039,7 +2177,7 @@
if device and device[ 'available' ]:
result = main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="add switch succesful",
+ onpass="add switch successful",
onfail="Failed to add switch?" )
def CASE13( self, main ):
@@ -2048,37 +2186,22 @@
"""
import os
import time
- # TODO: make use of this elsewhere
- ips = []
- ips.append( ONOS1Ip )
- ips.append( ONOS2Ip )
- ips.append( ONOS3Ip )
- ips.append( ONOS4Ip )
- ips.append( ONOS5Ip )
- ips.append( ONOS6Ip )
- ips.append( ONOS7Ip )
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# printing colors to terminal
- colors = {}
- colors[ 'cyan' ] = '\033[96m'
- colors[ 'purple' ] = '\033[95m'
- colors[ 'blue' ] = '\033[94m'
- colors[ 'green' ] = '\033[92m'
- colors[ 'yellow' ] = '\033[93m'
- colors[ 'red' ] = '\033[91m'
- colors[ 'end' ] = '\033[0m'
+ colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
+ 'blue': '\033[94m', 'green': '\033[92m',
+ 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
description = "Test Cleanup"
main.log.report( description )
main.case( description )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
- main.step( "Checking ONOS Logs for errors" )
- for i in range( 7 ):
- print colors[ 'purple' ] + "Checking logs for errors on " + \
- "ONOS" + str( i + 1 ) + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( ips[ i ] )
-
main.step( "Copying MN pcap and ONOS log files to test station" )
testname = main.TEST
teststationUser = main.params[ 'TESTONUSER' ]
@@ -2094,14 +2217,13 @@
# NOTE: must end in /
dstDir = "~/packet_captures/"
for f in logFiles:
- for i in range( 7 ):
- main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
- logFolder + f + " " +
+ for node in nodes:
+ main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
+ ":" + logFolder + f + " " +
teststationUser + "@" +
teststationIP + ":" +
dstDir + str( testname ) +
- "-ONOS" + str( i + 1 ) + "-" +
- f )
+ "-" + node.name + "-" + f )
main.ONOSbench.handle.expect( "\$" )
# std*.log's
@@ -2111,18 +2233,26 @@
# NOTE: must end in /
dstDir = "~/packet_captures/"
for f in logFiles:
- for i in range( 7 ):
- main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
- logFolder + f + " " +
+ for node in nodes:
+ main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
+ ":" + logFolder + f + " " +
teststationUser + "@" +
teststationIP + ":" +
dstDir + str( testname ) +
- "-ONOS" + str( i + 1 ) + "-" +
- f )
+ "-" + node.name + "-" + f )
main.ONOSbench.handle.expect( "\$" )
# sleep so scp can finish
time.sleep( 10 )
+
+ main.step( "Stopping Mininet" )
main.Mininet1.stopNet()
+
+ main.step( "Checking ONOS Logs for errors" )
+ for node in nodes:
+ print colors[ 'purple' ] + "Checking logs for errors on " + \
+ node.name + ":" + colors[ 'end' ]
+ print main.ONOSbench.checkLogs( node.ip_address )
+
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
@@ -2135,22 +2265,29 @@
"""
start election app on all onos nodes
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
leaderResult = main.TRUE
# install app on onos 1
main.log.info( "Install leadership election app" )
main.ONOScli1.featureInstall( "onos-app-election" )
+ leader = nodes[0].ip_address
# wait for election
# check for leader
- leader = main.ONOScli1.electionTestLeader()
+ leader1 = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
- if leader == ONOS1Ip:
+ if leader1 == leader:
# all is well
pass
- elif leader is None:
+ elif leader1 is None:
# No leader elected
main.log.report( "No leader was elected" )
leaderResult = main.FALSE
- elif leader == main.FALSE:
+ elif leader1 == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
@@ -2161,19 +2298,16 @@
# error in response
main.log.report(
"Unexpected response from electionTestLeader function:'" +
- str( leader ) +
- "'" )
+ str( leader1 ) + "'" )
leaderResult = main.FALSE
# install on other nodes and check for leader.
- # Should be onos1 and each app should show the same leader
- for controller in range( 2, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- node.featureInstall( "onos-app-election" )
- leaderN = node.electionTestLeader()
+ # Leader should be ONOS1 and each app should show the same leader
+ for cli in CLIs[ 1: ]:
+ cli.featureInstall( "onos-app-election" )
+ leaderN = cli.electionTestLeader()
# verify leader is ONOS1
- if leaderN == ONOS1Ip:
+ if leaderN == leader:
# all is well
pass
elif leaderN == main.FALSE:
@@ -2186,8 +2320,7 @@
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( "ONOS" + str( controller ) + " sees " +
- str( leaderN ) +
+ main.log.report( cli.name + " sees " + str( leaderN ) +
" as the leader of the election app. Leader" +
" should be " +
str( leader ) )
@@ -2205,6 +2338,12 @@
"""
Check that Leadership Election is still functional
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
main.log.report( description )
@@ -2213,28 +2352,18 @@
leader = main.ONOScli1.electionTestLeader()
# TODO: do some sanity checking on leader before using it
withdrawResult = main.FALSE
- if leader == ONOS1Ip:
- oldLeader = getattr( main, "ONOScli1" )
- elif leader == ONOS2Ip:
- oldLeader = getattr( main, "ONOScli2" )
- elif leader == ONOS3Ip:
- oldLeader = getattr( main, "ONOScli3" )
- elif leader == ONOS4Ip:
- oldLeader = getattr( main, "ONOScli4" )
- elif leader == ONOS5Ip:
- oldLeader = getattr( main, "ONOScli5" )
- elif leader == ONOS6Ip:
- oldLeader = getattr( main, "ONOScli6" )
- elif leader == ONOS7Ip:
- oldLeader = getattr( main, "ONOScli7" )
- elif leader is None or leader == main.FALSE:
+ if leader is None or leader == main.FALSE:
main.log.report(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
oldLeader = None
+ for i in range( len( CLIs ) ):
+ if leader == nodes[ i ].ip_address:
+ oldLeader = CLIs[ i ]
+ break
else:
- main.log.error( "Leader election --- why am I HERE?!?")
+ main.log.error( "Leader election, could not find current leader" )
if oldLeader:
withdrawResult = oldLeader.electionTestWithdraw()
utilities.assert_equals(
@@ -2245,21 +2374,17 @@
main.step( "Make sure new leader is elected" )
leaderList = []
- for controller in range( 1, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- leaderList.append( node.electionTestLeader() )
- for leaderN in leaderList:
+ for cli in CLIs:
+ leaderN = cli.electionTestLeader()
+ leaderList.append( leaderN )
if leaderN == leader:
- main.log.report(
- "ONOS" + str( controller ) +
- " still sees " + str( leader ) +
- " as leader after they withdrew" )
+ main.log.report( cli.name + " still sees " + str( leader ) +
+ " as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
- # means the app isn't loaded
+ # means the app isn't loaded
main.log.report( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
@@ -2276,6 +2401,7 @@
for n in range( len( leaderList ) ):
main.log.report( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
+ leaderResult = leaderResult and consistentLeader
if leaderResult:
main.log.report( "Leadership election tests passed( consistent " +
"view of leader across listeners and a new " +
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 3e7de57..ba2b113 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -49,7 +49,7 @@
main.case( "Setting up test environment" )
# TODO: save all the timers and output them for plotting
- # load some vairables from the params file
+ # load some variables from the params file
PULLCODE = False
if main.params[ 'Git' ] == 'True':
PULLCODE = True
@@ -57,38 +57,34 @@
cellName = main.params[ 'ENV' ][ 'cellName' ]
# set global variables
- global ONOS1Ip
global ONOS1Port
- global ONOS2Ip
global ONOS2Port
- global ONOS3Ip
global ONOS3Port
- global ONOS4Ip
global ONOS4Port
- global ONOS5Ip
global ONOS5Port
- global ONOS6Ip
global ONOS6Port
- global ONOS7Ip
global ONOS7Port
global numControllers
-
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
- ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
- ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
- ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
- ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
- ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
- ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
- ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
- ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
- ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
- ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
numControllers = int( main.params[ 'num_controllers' ] )
+ # FIXME: just get controller port from params?
+ # TODO: do we really need all these?
+ ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
+ ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
+ ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
+ ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
+ ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
+ ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
+ ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
+
+ global CLIs
+ CLIs = []
+ global nodes
+ nodes = []
+ for i in range( 1, numControllers + 1 ):
+ CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+ nodes.append( getattr( main, 'ONOS' + str( i ) ) )
+
main.step( "Applying cell variable to environment" )
cellResult = main.ONOSbench.setCell( cellName )
verifyResult = main.ONOSbench.verifyCell()
@@ -96,14 +92,10 @@
# FIXME:this is short term fix
main.log.report( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
+
main.log.report( "Uninstalling ONOS" )
- main.ONOSbench.onosUninstall( ONOS1Ip )
- main.ONOSbench.onosUninstall( ONOS2Ip )
- main.ONOSbench.onosUninstall( ONOS3Ip )
- main.ONOSbench.onosUninstall( ONOS4Ip )
- main.ONOSbench.onosUninstall( ONOS5Ip )
- main.ONOSbench.onosUninstall( ONOS6Ip )
- main.ONOSbench.onosUninstall( ONOS7Ip )
+ for node in nodes:
+ main.ONOSbench.onosUninstall( node.ip_address )
cleanInstallResult = main.TRUE
gitPullResult = main.TRUE
@@ -116,6 +108,8 @@
main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
+ if gitPullResult == main.ERROR:
+ main.log.error( "Error pulling git branch" )
main.step( "Using mvn clean & install" )
cleanInstallResult = main.ONOSbench.cleanInstall()
@@ -128,76 +122,38 @@
packageResult = main.ONOSbench.onosPackage()
main.step( "Installing ONOS package" )
- onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS1Ip )
- onos2InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS2Ip )
- onos3InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS3Ip )
- onos4InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS4Ip )
- onos5InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS5Ip )
- onos6InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS6Ip )
- onos7InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS7Ip )
- onosInstallResult = onos1InstallResult and onos2InstallResult\
- and onos3InstallResult and onos4InstallResult\
- and onos5InstallResult and onos6InstallResult\
- and onos7InstallResult
+ onosInstallResult = main.TRUE
+ for node in nodes:
+ tmpResult = main.ONOSbench.onosInstall( options="-f",
+ node=node.ip_address )
+ onosInstallResult = onosInstallResult and tmpResult
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
- onos1Isup = main.ONOSbench.isup( ONOS1Ip )
- if not onos1Isup:
- main.log.report( "ONOS1 didn't start!" )
- main.ONOSbench.onosStop( ONOS1Ip )
- main.ONOSbench.onosStart( ONOS1Ip )
- onos2Isup = main.ONOSbench.isup( ONOS2Ip )
- if not onos2Isup:
- main.log.report( "ONOS2 didn't start!" )
- main.ONOSbench.onosStop( ONOS2Ip )
- main.ONOSbench.onosStart( ONOS2Ip )
- onos3Isup = main.ONOSbench.isup( ONOS3Ip )
- if not onos3Isup:
- main.log.report( "ONOS3 didn't start!" )
- main.ONOSbench.onosStop( ONOS3Ip )
- main.ONOSbench.onosStart( ONOS3Ip )
- onos4Isup = main.ONOSbench.isup( ONOS4Ip )
- if not onos4Isup:
- main.log.report( "ONOS4 didn't start!" )
- main.ONOSbench.onosStop( ONOS4Ip )
- main.ONOSbench.onosStart( ONOS4Ip )
- onos5Isup = main.ONOSbench.isup( ONOS5Ip )
- if not onos5Isup:
- main.log.report( "ONOS5 didn't start!" )
- main.ONOSbench.onosStop( ONOS5Ip )
- main.ONOSbench.onosStart( ONOS5Ip )
- onos6Isup = main.ONOSbench.isup( ONOS6Ip )
- if not onos6Isup:
- main.log.report( "ONOS6 didn't start!" )
- main.ONOSbench.onosStop( ONOS6Ip )
- main.ONOSbench.onosStart( ONOS6Ip )
- onos7Isup = main.ONOSbench.isup( ONOS7Ip )
- if not onos7Isup:
- main.log.report( "ONOS7 didn't start!" )
- main.ONOSbench.onosStop( ONOS7Ip )
- main.ONOSbench.onosStart( ONOS7Ip )
- onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
- and onos4Isup and onos5Isup and onos6Isup and onos7Isup
+ onosIsupResult = main.TRUE
+ for node in nodes:
+ started = main.ONOSbench.isup( node.ip_address )
+ if not started:
+ main.log.report( node.name + " didn't start!" )
+ main.ONOSbench.onosStop( node.ip_address )
+ main.ONOSbench.onosStart( node.ip_address )
+ onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
- cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
- cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
- cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
- cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
- cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
- cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
- cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
- cliResults = cliResult1 and cliResult2 and cliResult3 and\
- cliResult4 and cliResult5 and cliResult6 and cliResult7
+ main.log.step( "Starting ONOS CLI sessions" )
+ cliResults = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].startOnosCli,
+ name="startOnosCli-" + str( i ),
+ args=[nodes[i].ip_address] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ cliResults = cliResults and t.result
main.step( "Start Packet Capture MN" )
main.Mininet2.startTcpdump(
@@ -223,22 +179,36 @@
Assign mastership to controllers
"""
import re
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert ONOS1Port, "ONOS1Port not defined"
+ assert ONOS2Port, "ONOS2Port not defined"
+ assert ONOS3Port, "ONOS3Port not defined"
+ assert ONOS4Port, "ONOS4Port not defined"
+ assert ONOS5Port, "ONOS5Port not defined"
+ assert ONOS6Port, "ONOS6Port not defined"
+ assert ONOS7Port, "ONOS7Port not defined"
main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
main.step( "Assign switches to controllers" )
+ # TODO: rewrite this function to take lists of ips and ports?
+ # or list of tuples?
for i in range( 1, 29 ):
main.Mininet1.assignSwController(
sw=str( i ),
count=numControllers,
- ip1=ONOS1Ip, port1=ONOS1Port,
- ip2=ONOS2Ip, port2=ONOS2Port,
- ip3=ONOS3Ip, port3=ONOS3Port,
- ip4=ONOS4Ip, port4=ONOS4Port,
- ip5=ONOS5Ip, port5=ONOS5Port,
- ip6=ONOS6Ip, port6=ONOS6Port,
- ip7=ONOS7Ip, port7=ONOS7Port )
+ ip1=nodes[ 0 ].ip_address, port1=ONOS1Port,
+ ip2=nodes[ 1 ].ip_address, port2=ONOS2Port,
+ ip3=nodes[ 2 ].ip_address, port3=ONOS3Port,
+ ip4=nodes[ 3 ].ip_address, port4=ONOS4Port,
+ ip5=nodes[ 4 ].ip_address, port5=ONOS5Port,
+ ip6=nodes[ 5 ].ip_address, port6=ONOS6Port,
+ ip7=nodes[ 6 ].ip_address, port7=ONOS7Port )
mastershipCheck = main.TRUE
for i in range( 1, 29 ):
@@ -247,16 +217,11 @@
main.log.info( str( response ) )
except Exception:
main.log.info( repr( response ) )
- if re.search( "tcp:" + ONOS1Ip, response )\
- and re.search( "tcp:" + ONOS2Ip, response )\
- and re.search( "tcp:" + ONOS3Ip, response )\
- and re.search( "tcp:" + ONOS4Ip, response )\
- and re.search( "tcp:" + ONOS5Ip, response )\
- and re.search( "tcp:" + ONOS6Ip, response )\
- and re.search( "tcp:" + ONOS7Ip, response ):
- mastershipCheck = mastershipCheck and main.TRUE
- else:
- mastershipCheck = main.FALSE
+ for node in nodes:
+ if re.search( "tcp:" + node.ip_address, response ):
+ mastershipCheck = mastershipCheck and main.TRUE
+ else:
+ mastershipCheck = main.FALSE
if mastershipCheck == main.TRUE:
main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
@@ -264,19 +229,20 @@
actual=mastershipCheck,
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
-
+ #FIXME: turning off because of ONOS-1286
# Manually assign mastership to the controller we want
roleCall = main.TRUE
roleCheck = main.TRUE
try:
# Assign switch
+ ip = nodes[ 0 ].ip_address # ONOS1
deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
assert deviceId, "No device id for s1 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS1Ip )
+ ip )
# Check assignment
- if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
@@ -286,21 +252,22 @@
assert deviceId, "No device id for s28 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS1Ip )
+ ip )
# Check assignment
- if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 1 ].ip_address # ONOS2
# Assign switch
deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
assert deviceId, "No device id for s2 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS2Ip )
+ ip )
# Check assignment
- if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
@@ -310,21 +277,22 @@
assert deviceId, "No device id for s3 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS2Ip )
+ ip )
# Check assignment
- if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 2 ].ip_address # ONOS3
# Assign switch
deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
assert deviceId, "No device id for s5 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS3Ip )
+ ip )
# Check assignment
- if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
@@ -334,65 +302,68 @@
assert deviceId, "No device id for s6 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS3Ip )
+ ip )
# Check assignment
- if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 3 ].ip_address # ONOS4
# Assign switch
deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
assert deviceId, "No device id for s4 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS4Ip )
+ ip )
# Check assignment
- if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 4 ].ip_address # ONOS5
for i in range( 8, 18 ):
dpid = '3' + str( i ).zfill( 3 )
deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
assert deviceId, "No device id for s%i in ONOS" % i
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS5Ip )
+ ip )
# Check assignment
- if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 5 ].ip_address # ONOS6
deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
assert deviceId, "No device id for s7 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS6Ip )
+ ip )
# Check assignment
- if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 6 ].ip_address # ONOS7
for i in range( 18, 28 ):
dpid = '6' + str( i ).zfill( 3 )
deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
assert deviceId, "No device id for s%i in ONOS" % i
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS7Ip )
+ ip )
# Check assignment
- if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
except ( AttributeError, AssertionError ):
main.log.exception( "Something is wrong with ONOS device view" )
main.log.info( main.ONOScli1.devices() )
-
utilities.assert_equals(
expect=main.TRUE,
actual=roleCall,
@@ -417,6 +388,11 @@
"""
import time
import json
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
main.log.report( "Adding host intents" )
main.case( "Adding host Intents" )
@@ -425,39 +401,50 @@
# install onos-app-fwd
main.log.info( "Install reactive forwarding app" )
- main.ONOScli1.featureInstall( "onos-app-fwd" )
- main.ONOScli2.featureInstall( "onos-app-fwd" )
- main.ONOScli3.featureInstall( "onos-app-fwd" )
- main.ONOScli4.featureInstall( "onos-app-fwd" )
- main.ONOScli5.featureInstall( "onos-app-fwd" )
- main.ONOScli6.featureInstall( "onos-app-fwd" )
- main.ONOScli7.featureInstall( "onos-app-fwd" )
+ appResults = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].featureInstall,
+ name="featureInstall-" + str( i ),
+ args=["onos-app-fwd"] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ appResults = appResults and t.result
# REACTIVE FWD test
pingResult = main.FALSE
- time1 = time.time()
- pingResult = main.Mininet1.pingall()
- utilities.assert_equals(
- expect=main.TRUE,
- actual=pingResult,
- onpass="Reactive Pingall test passed",
- onfail="Reactive Pingall failed, one or more ping pairs failed" )
- time2 = time.time()
- main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
+ for i in range(2): # Retry if pingall fails first time
+ time1 = time.time()
+ pingResult = main.Mininet1.pingall()
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=pingResult,
+ onpass="Reactive Pingall test passed",
+ onfail="Reactive Pingall failed, one or more ping pairs failed" )
+ time2 = time.time()
+ main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
# uninstall onos-app-fwd
main.log.info( "Uninstall reactive forwarding app" )
- main.ONOScli1.featureUninstall( "onos-app-fwd" )
- main.ONOScli2.featureUninstall( "onos-app-fwd" )
- main.ONOScli3.featureUninstall( "onos-app-fwd" )
- main.ONOScli4.featureUninstall( "onos-app-fwd" )
- main.ONOScli5.featureUninstall( "onos-app-fwd" )
- main.ONOScli6.featureUninstall( "onos-app-fwd" )
- main.ONOScli7.featureUninstall( "onos-app-fwd" )
- # timeout for fwd flows
- time.sleep( 10 )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].featureUninstall,
+ name="featureUninstall-" + str( i ),
+ args=["onos-app-fwd"] )
+ threads.append( t )
+ t.start()
- main.step( "Add host intents" )
+ for t in threads:
+ t.join()
+ appResults = appResults and t.result
+
+ # timeout for fwd flows
+ time.sleep( 11 )
+
+ main.step( "Add host intents" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -479,42 +466,50 @@
host1Id = host1Dict.get( 'id', None )
host2Id = host2Dict.get( 'id', None )
if host1Id and host2Id:
- # Changed onos node to test something
- tmpId = main.ONOScli4.addHostIntent(
- host1Id,
- host2Id )
+ nodeNum = ( i % 7 )
+ tmpId = CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
if tmpId:
main.log.info( "Added intent with id: " + tmpId )
intentIds.append( tmpId )
else:
- main.log.error( "addHostIntent reutrned None" )
+ main.log.error( "addHostIntent returned: " +
+ repr( tmpId ) )
else:
- main.log.error( "Error, getHost() failed" )
- main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.error( "Error, getHost() failed for h" + str( i ) +
+ " and/or h" + str( i + 10 ) )
+ hosts = CLIs[ 0 ].hosts()
+ main.log.warn( "Hosts output: " )
+ try:
+ main.log.warn( json.dumps( json.loads( hosts ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( hosts ) )
hostResult = main.FALSE
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
main.log.info( "Intents in ONOS: " + str( onosIds ) )
for intent in intentIds:
if intent in onosIds:
- pass # intent submitted is still in onos
+ pass # intent submitted is in onos
else:
intentAddResult = False
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
- installedCheck = True
+ installedCheck = True
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
- for intent in json.loads( intents ): # Iter through intents of a node
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents" )
# add submitted intents not in the store
tmplist = [ i for i, s in intentStates ]
missingIntents = False
@@ -527,17 +522,62 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- # for node in nodes:
- # node.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
pendingMap = main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
+
intentAddResult = bool( pingResult and hostResult and intentAddResult
and not missingIntents and installedCheck )
utilities.assert_equals(
@@ -559,13 +599,16 @@
intentStates = []
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
- for intent in json.loads( intents ):
- # Iter through intents of a node
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ # Iter through intents of a node
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents" )
# add submitted intents not in the store
tmplist = [ i for i, s in intentStates ]
for i in intentIds:
@@ -576,20 +619,73 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
def CASE4( self, main ):
"""
Ping across added host intents
"""
import json
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
description = " Ping across added host intents"
main.log.report( description )
main.case( description )
@@ -608,11 +704,15 @@
main.log.report(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
- main.log.warn( "ONSO1 intents: " )
- main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
if PingResult == main.TRUE:
main.log.report(
"Intents have been installed correctly and verified by pings" )
@@ -630,28 +730,79 @@
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
# Iter through intents of a node
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
+
if not installedCheck:
- main.log.info( "Waiting 60 seconds to see if intent states change" )
+ main.log.info( "Waiting 60 seconds to see if the state of " +
+ "intents change" )
time.sleep( 60 )
# Print the intent states
intents = main.ONOScli1.intents()
@@ -659,32 +810,87 @@
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
# Iter through intents of a node
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
def CASE5( self, main ):
"""
Reading state of ONOS
"""
import json
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
@@ -692,344 +898,231 @@
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
- # We can then compare them with eachother and also with past states
+ # We can then compare them with each other and also with past states
- main.step( "Get the Mastership of each switch from each controller" )
+ main.step( "Check that each switch has a master" )
global mastershipState
- mastershipState = []
+ mastershipState = '[]'
# Assert that each device has a master
- ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
- ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
- ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
- ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
- ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
- ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
- ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
- rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
- ONOS3MasterNotNull and ONOS4MasterNotNull and\
- ONOS5MasterNotNull and ONOS6MasterNotNull and\
- ONOS7MasterNotNull
+ rolesNotNull = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].rolesNotNull,
+ name="rolesNotNull-" + str( i ),
+ args=[] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ rolesNotNull = rolesNotNull and t.result
utilities.assert_equals(
expect=main.TRUE,
actual=rolesNotNull,
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
- ONOS1Mastership = main.ONOScli1.roles()
- ONOS2Mastership = main.ONOScli2.roles()
- ONOS3Mastership = main.ONOScli3.roles()
- ONOS4Mastership = main.ONOScli4.roles()
- ONOS5Mastership = main.ONOScli5.roles()
- ONOS6Mastership = main.ONOScli6.roles()
- ONOS7Mastership = main.ONOScli7.roles()
- if "Error" in ONOS1Mastership or not ONOS1Mastership\
- or "Error" in ONOS2Mastership or not ONOS2Mastership\
- or "Error" in ONOS3Mastership or not ONOS3Mastership\
- or "Error" in ONOS4Mastership or not ONOS4Mastership\
- or "Error" in ONOS5Mastership or not ONOS5Mastership\
- or "Error" in ONOS6Mastership or not ONOS6Mastership\
- or "Error" in ONOS7Mastership or not ONOS7Mastership:
- main.log.report( "Error in getting ONOS roles" )
- main.log.warn(
- "ONOS1 mastership response: " +
- repr( ONOS1Mastership ) )
- main.log.warn(
- "ONOS2 mastership response: " +
- repr( ONOS2Mastership ) )
- main.log.warn(
- "ONOS3 mastership response: " +
- repr( ONOS3Mastership ) )
- main.log.warn(
- "ONOS4 mastership response: " +
- repr( ONOS4Mastership ) )
- main.log.warn(
- "ONOS5 mastership response: " +
- repr( ONOS5Mastership ) )
- main.log.warn(
- "ONOS6 mastership response: " +
- repr( ONOS6Mastership ) )
- main.log.warn(
- "ONOS7 mastership response: " +
- repr( ONOS7Mastership ) )
- consistentMastership = main.FALSE
- elif ONOS1Mastership == ONOS2Mastership\
- and ONOS1Mastership == ONOS3Mastership\
- and ONOS1Mastership == ONOS4Mastership\
- and ONOS1Mastership == ONOS5Mastership\
- and ONOS1Mastership == ONOS6Mastership\
- and ONOS1Mastership == ONOS7Mastership:
- mastershipState = ONOS1Mastership
- consistentMastership = main.TRUE
+ main.step( "Get the Mastership of each switch from each controller" )
+ ONOSMastership = []
+ mastershipCheck = main.FALSE
+ consistentMastership = True
+ rolesResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].roles,
+ name="roles-" + str( i ),
+ args=[] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSMastership.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " roles" )
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " mastership response: " +
+ repr( ONOSMastership[i] ) )
+ rolesResults = False
+ utilities.assert_equals(
+ expect=True,
+ actual=rolesResults,
+ onpass="No error in reading roles output",
+ onfail="Error in reading roles from ONOS" )
+
+ main.step( "Check for consistency in roles from each controller" )
+ if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
main.log.report(
"Switch roles are consistent across all ONOS nodes" )
else:
- main.log.warn(
- "ONOS1 roles: ",
- json.dumps(
- json.loads( ONOS1Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS2 roles: ",
- json.dumps(
- json.loads( ONOS2Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS3 roles: ",
- json.dumps(
- json.loads( ONOS3Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS4 roles: ",
- json.dumps(
- json.loads( ONOS4Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS5 roles: ",
- json.dumps(
- json.loads( ONOS5Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS6 roles: ",
- json.dumps(
- json.loads( ONOS6Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS7 roles: ",
- json.dumps(
- json.loads( ONOS7Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- consistentMastership = main.FALSE
+ consistentMastership = False
utilities.assert_equals(
- expect=main.TRUE,
+ expect=True,
actual=consistentMastership,
onpass="Switch roles are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of switch roles" )
+ if rolesResults and not consistentMastership:
+ for i in range( numControllers ):
+ try:
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " roles: ",
+ json.dumps(
+ json.loads( ONOSMastership[ i ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( ONOSMastership[ i ] ) )
+ elif rolesResults and consistentMastership:
+ mastershipCheck = main.TRUE
+ mastershipState = ONOSMastership[ 0 ]
+
main.step( "Get the intents from each controller" )
global intentState
intentState = []
- ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
- ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
- ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
- ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
- ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
- ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
- ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
+ ONOSIntents = []
intentCheck = main.FALSE
- if "Error" in ONOS1Intents or not ONOS1Intents\
- or "Error" in ONOS2Intents or not ONOS2Intents\
- or "Error" in ONOS3Intents or not ONOS3Intents\
- or "Error" in ONOS4Intents or not ONOS4Intents\
- or "Error" in ONOS5Intents or not ONOS5Intents\
- or "Error" in ONOS6Intents or not ONOS6Intents\
- or "Error" in ONOS7Intents or not ONOS7Intents:
- main.log.report( "Error in getting ONOS intents" )
- main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
- main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
- main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
- main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
- main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
- main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
- main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
- elif ONOS1Intents == ONOS2Intents\
- and ONOS1Intents == ONOS3Intents\
- and ONOS1Intents == ONOS4Intents\
- and ONOS1Intents == ONOS5Intents\
- and ONOS1Intents == ONOS6Intents\
- and ONOS1Intents == ONOS7Intents:
- intentState = ONOS1Intents
- intentCheck = main.TRUE
- main.log.report( "Intents are consistent across all ONOS nodes" )
- else:
- main.log.warn(
- "ONOS1 intents: ",
- json.dumps(
- json.loads( ONOS1Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS2 intents: ",
- json.dumps(
- json.loads( ONOS2Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS3 intents: ",
- json.dumps(
- json.loads( ONOS3Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS4 intents: ",
- json.dumps(
- json.loads( ONOS4Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS5 intents: ",
- json.dumps(
- json.loads( ONOS5Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS6 intents: ",
- json.dumps(
- json.loads( ONOS6Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS7 intents: ",
- json.dumps(
- json.loads( ONOS7Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
+ consistentIntents = True
+ intentsResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].intents,
+ name="intents-" + str( i ),
+ args=[],
+ kwargs={ 'jsonFormat': True } )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSIntents.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " intents" )
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
+ repr( ONOSIntents[ i ] ) )
+ intentsResults = False
utilities.assert_equals(
- expect=main.TRUE,
- actual=intentCheck,
+ expect=True,
+ actual=intentsResults,
+ onpass="No error in reading intents output",
+ onfail="Error in reading intents from ONOS" )
+
+ main.step( "Check for consistency in Intents from each controller" )
+ if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
+ main.log.report( "Intents are consistent across all ONOS " +
+ "nodes" )
+ else:
+ consistentIntents = False
+ main.log.report( "Intents not consistent" )
+ utilities.assert_equals(
+ expect=True,
+ actual=consistentIntents,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ if intentsResults and not consistentIntents:
+ n = len(ONOSIntents)
+ main.log.warn( "ONOS" + str( n ) + " intents: " )
+ main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ for i in range( numControllers ):
+ if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
+ main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ else:
+ main.log.warn( nodes[ i ].name + " intents match ONOS" +
+ str( n ) + " intents" )
+ elif intentsResults and consistentIntents:
+ intentCheck = main.TRUE
+ intentState = ONOSIntents[ 0 ]
+
main.step( "Get the flows from each controller" )
global flowState
flowState = []
+ ONOSFlows = []
+ ONOSFlowsJson = []
flowCheck = main.FALSE
- try:
- ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
- ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
- ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
- ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
- ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
- ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
- ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
- assert ONOS1Flows, "ONOS1 Flows should not be empty"
- assert ONOS2Flows, "ONOS2 Flows should not be empty"
- assert ONOS3Flows, "ONOS3 Flows should not be empty"
- assert ONOS4Flows, "ONOS4 Flows should not be empty"
- assert ONOS5Flows, "ONOS5 Flows should not be empty"
- assert ONOS6Flows, "ONOS6 Flows should not be empty"
- assert ONOS7Flows, "ONOS7 Flows should not be empty"
- assert "Error" not in ONOS1Flows, "ONOS1 Flows contains 'Error'"
- assert "Error" not in ONOS2Flows, "ONOS2 Flows contains 'Error'"
- assert "Error" not in ONOS3Flows, "ONOS3 Flows contains 'Error'"
- assert "Error" not in ONOS4Flows, "ONOS4 Flows contains 'Error'"
- assert "Error" not in ONOS5Flows, "ONOS5 Flows contains 'Error'"
- assert "Error" not in ONOS6Flows, "ONOS6 Flows contains 'Error'"
- assert "Error" not in ONOS7Flows, "ONOS7 Flows contains 'Error'"
- ONOS1FlowsJson = json.loads( ONOS1Flows )
- ONOS2FlowsJson = json.loads( ONOS2Flows )
- ONOS3FlowsJson = json.loads( ONOS3Flows )
- ONOS4FlowsJson = json.loads( ONOS4Flows )
- ONOS5FlowsJson = json.loads( ONOS5Flows )
- ONOS6FlowsJson = json.loads( ONOS6Flows )
- ONOS7FlowsJson = json.loads( ONOS7Flows )
- except ( ValueError, AssertionError ): # From json.loads, or asserts
- main.log.exception( "One or more 'flows' responses from " +
- "ONOS couldn't be decoded." )
- main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
- main.log.warn( "ONOS2 flows repsponse: " + ONOS2Flows )
- main.log.warn( "ONOS3 flows repsponse: " + ONOS3Flows )
- main.log.warn( "ONOS4 flows repsponse: " + ONOS4Flows )
- main.log.warn( "ONOS5 flows repsponse: " + ONOS5Flows )
- main.log.warn( "ONOS6 flows repsponse: " + ONOS6Flows )
- main.log.warn( "ONOS7 flows repsponse: " + ONOS7Flows )
- else: # No exceptions
- if len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
- # TODO: Do a better check, maybe compare flows on switches?
- # NOTE Possible issue with this not always being set?
- flowState = ONOS1Flows
- flowCheck = main.TRUE
- main.log.report( "Flow count is consistent across all" +
- " ONOS nodes" )
+ consistentFlows = True
+ flowsResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].flows,
+ name="flows-" + str( i ),
+ args=[],
+ kwargs={ 'jsonFormat': True } )
+ threads.append( t )
+ t.start()
+
+ time.sleep(30)
+ for t in threads:
+ t.join()
+ result = t.result
+ ONOSFlows.append( result )
+
+ for i in range( numControllers ):
+ num = str( i + 1 )
+ if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
+ main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.warn( "ONOS" + num + " flows response: " +
+ repr( ONOSFlows[ i ] ) )
+ flowsResults = False
+ ONOSFlowsJson.append( None )
else:
- main.log.warn( "ONOS1 flows: " +
- json.dumps( ONOS1FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS2 flows: " +
- json.dumps( ONOS2FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS3 flows: " +
- json.dumps( ONOS3FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS4 flows: " +
- json.dumps( ONOS4FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS5 flows: " +
- json.dumps( ONOS5FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS6 flows: " +
- json.dumps( ONOS6FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS7 flows: " +
- json.dumps( ONOS7FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ try:
+ ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
+ except ( ValueError, TypeError ):
+ # FIXME: change this to log.error?
+ main.log.exception( "Error in parsing ONOS" + num +
+ " response as json." )
+ main.log.error( repr( ONOSFlows[ i ] ) )
+ ONOSFlowsJson.append( None )
+ flowsResults = False
utilities.assert_equals(
- expect=main.TRUE,
- actual=flowCheck,
+ expect=True,
+ actual=flowsResults,
+ onpass="No error in reading flows output",
+ onfail="Error in reading flows from ONOS" )
+
+ main.step( "Check for consistency in Flows from each controller" )
+ tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
+ if all( tmp ):
+ main.log.report( "Flow count is consistent across all ONOS nodes" )
+ else:
+ consistentFlows = False
+ utilities.assert_equals(
+ expect=True,
+ actual=consistentFlows,
onpass="The flow count is consistent across all ONOS nodes",
onfail="ONOS nodes have different flow counts" )
+ if flowsResults and not consistentFlows:
+ for i in range( numControllers ):
+ try:
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " flows: " +
+ json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
+ indent=4, separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " flows: " +
+ repr( ONOSFlows[ i ] ) )
+ elif flowsResults and consistentFlows:
+ flowCheck = main.TRUE
+ flowState = ONOSFlows[ 0 ]
+
main.step( "Get the OF Table entries" )
global flows
flows = []
@@ -1084,64 +1177,80 @@
main.step( "Create TestONTopology object" )
ctrls = []
- count = 1
- while True:
- temp = ()
- if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
- temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
- temp = temp + ( "ONOS" + str( count ), )
- temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
- temp = temp + \
- ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
- ctrls.append( temp )
- count = count + 1
- else:
- break
- MNTopo = TestONTopology(
- main.Mininet1,
- ctrls ) # can also add Intent API info for intent operations
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
main.step( "Collecting topology information from ONOS" )
devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].devices,
+ name="devices-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ devices.append( t.result )
hosts = []
- hosts.append( json.loads( main.ONOScli1.hosts() ) )
- hosts.append( json.loads( main.ONOScli2.hosts() ) )
- hosts.append( json.loads( main.ONOScli3.hosts() ) )
- hosts.append( json.loads( main.ONOScli4.hosts() ) )
- hosts.append( json.loads( main.ONOScli5.hosts() ) )
- hosts.append( json.loads( main.ONOScli6.hosts() ) )
- hosts.append( json.loads( main.ONOScli7.hosts() ) )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].hosts,
+ name="hosts-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ try:
+ hosts.append( json.loads( t.result ) )
+ except ( ValueError, TypeError ):
+ # FIXME: better handling of this, print which node
+ # Maybe use thread name?
+ main.log.exception( "Error parsing json output of hosts" )
+ # FIXME: should this be an empty json object instead?
+ hosts.append( None )
+
ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].ports,
+ name="ports-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ports.append( t.result )
links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].links,
+ name="links-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ links.append( t.result )
clusters = []
- clusters.append( main.ONOScli1.clusters() )
- clusters.append( main.ONOScli2.clusters() )
- clusters.append( main.ONOScli3.clusters() )
- clusters.append( main.ONOScli4.clusters() )
- clusters.append( main.ONOScli5.clusters() )
- clusters.append( main.ONOScli6.clusters() )
- clusters.append( main.ONOScli7.clusters() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].clusters,
+ name="clusters-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ clusters.append( t.result )
# Compare json objects for hosts and dataplane clusters
# hosts
@@ -1175,10 +1284,9 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host.get( 'ips', [] ) == []:
- main.log.error(
- "DEBUG:Error with host ips on controller" +
- controllerStr + ": " + str( host ) )
+ if not host.get( 'ips', [ ] ):
+ main.log.error( "DEBUG:Error with host ips on controller" +
+ controllerStr + ": " + str( host ) )
ipResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
@@ -1189,12 +1297,12 @@
# Strongly connected clusters of devices
consistentClustersResult = main.TRUE
for controller in range( len( clusters ) ):
+ controllerStr = str( controller + 1 )
if "Error" not in clusters[ controller ]:
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
- controllerStr +
+ main.log.report( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
@@ -1211,7 +1319,11 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
- numClusters = len( json.loads( clusters[ 0 ] ) )
+ try:
+ numClusters = len( json.loads( clusters[ 0 ] ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing clusters[0]: " +
+ repr( clusters[ 0 ] ) )
clusterResults = main.FALSE
if numClusters == 1:
clusterResults = main.TRUE
@@ -1230,8 +1342,7 @@
if devices[ controller ] or "Error" not in devices[ controller ]:
currentDevicesResult = main.Mininet1.compareSwitches(
MNTopo,
- json.loads(
- devices[ controller ] ) )
+ json.loads( devices[ controller ] ) )
else:
currentDevicesResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
@@ -1244,8 +1355,7 @@
if ports[ controller ] or "Error" not in ports[ controller ]:
currentPortsResult = main.Mininet1.comparePorts(
MNTopo,
- json.loads(
- ports[ controller ] ) )
+ json.loads( ports[ controller ] ) )
else:
currentPortsResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
@@ -1258,8 +1368,7 @@
if links[ controller ] or "Error" not in links[ controller ]:
currentLinksResult = main.Mininet1.compareLinks(
MNTopo,
- json.loads(
- links[ controller ] ) )
+ json.loads( links[ controller ] ) )
else:
currentLinksResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
@@ -1273,16 +1382,17 @@
portsResults = portsResults and currentPortsResult
linksResults = linksResults and currentLinksResult
- topoResult = devicesResults and portsResults and linksResults\
- and consistentHostsResult and consistentClustersResult\
- and clusterResults and ipResult
+ topoResult = ( devicesResults and portsResults and linksResults
+ and consistentHostsResult and consistentClustersResult
+ and clusterResults and ipResult )
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
finalAssert = main.TRUE
- finalAssert = finalAssert and topoResult and flowCheck \
- and intentCheck and consistentMastership and rolesNotNull
+ finalAssert = ( finalAssert and topoResult and flowCheck
+ and intentCheck and consistentMastership
+ and rolesNotNull )
utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
onpass="State check successful",
onfail="State check NOT successful" )
@@ -1292,29 +1402,35 @@
The Failure case.
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
main.log.report( "Killing 3 ONOS nodes" )
- main.log.case( "Restart minority of ONOS nodes" )
+ main.case( "Restart minority of ONOS nodes" )
# TODO: Randomize these nodes
- main.ONOSbench.onosKill( ONOS1Ip )
+ # TODO: use threads in this case
+ main.ONOSbench.onosKill( nodes[0].ip_address )
time.sleep( 10 )
- main.ONOSbench.onosKill( ONOS2Ip )
+ main.ONOSbench.onosKill( nodes[1].ip_address )
time.sleep( 10 )
- main.ONOSbench.onosKill( ONOS3Ip )
+ main.ONOSbench.onosKill( nodes[2].ip_address )
main.step( "Checking if ONOS is up yet" )
count = 0
onosIsupResult = main.FALSE
while onosIsupResult == main.FALSE and count < 10:
- onos1Isup = main.ONOSbench.isup( ONOS1Ip )
- onos2Isup = main.ONOSbench.isup( ONOS2Ip )
- onos3Isup = main.ONOSbench.isup( ONOS3Ip )
+ onos1Isup = main.ONOSbench.isup( nodes[0].ip_address )
+ onos2Isup = main.ONOSbench.isup( nodes[1].ip_address )
+ onos3Isup = main.ONOSbench.isup( nodes[2].ip_address )
onosIsupResult = onos1Isup and onos2Isup and onos3Isup
count = count + 1
# TODO: if it becomes an issue, we can retry this step a few times
- cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
- cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
- cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
+ cliResult1 = main.ONOScli1.startOnosCli( nodes[0].ip_address )
+ cliResult2 = main.ONOScli2.startOnosCli( nodes[1].ip_address )
+ cliResult3 = main.ONOScli3.startOnosCli( nodes[2].ip_address )
cliResults = cliResult1 and cliResult2 and cliResult3
# Grab the time of restart so we chan check how long the gossip
@@ -1330,108 +1446,103 @@
Check state after ONOS failure
"""
import json
- import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
main.case( "Running ONOS Constant State Tests" )
+ main.step( "Check that each switch has a master" )
# Assert that each device has a master
- ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
- ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
- ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
- ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
- ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
- ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
- ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
- rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
- ONOS3MasterNotNull and ONOS4MasterNotNull and\
- ONOS5MasterNotNull and ONOS6MasterNotNull and\
- ONOS7MasterNotNull
+ rolesNotNull = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].rolesNotNull,
+ name="rolesNotNull-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ rolesNotNull = rolesNotNull and t.result
utilities.assert_equals(
expect=main.TRUE,
actual=rolesNotNull,
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
- main.step( "Check if switch roles are consistent across all nodes" )
- ONOS1Mastership = main.ONOScli1.roles()
- ONOS2Mastership = main.ONOScli2.roles()
- ONOS3Mastership = main.ONOScli3.roles()
- ONOS4Mastership = main.ONOScli4.roles()
- ONOS5Mastership = main.ONOScli5.roles()
- ONOS6Mastership = main.ONOScli6.roles()
- ONOS7Mastership = main.ONOScli7.roles()
- if "Error" in ONOS1Mastership or not ONOS1Mastership\
- or "Error" in ONOS2Mastership or not ONOS2Mastership\
- or "Error" in ONOS3Mastership or not ONOS3Mastership\
- or "Error" in ONOS4Mastership or not ONOS4Mastership\
- or "Error" in ONOS5Mastership or not ONOS5Mastership\
- or "Error" in ONOS6Mastership or not ONOS6Mastership\
- or "Error" in ONOS7Mastership or not ONOS7Mastership:
- main.log.error( "Error in getting ONOS mastership" )
- main.log.warn( "ONOS1 mastership response: " +
- repr( ONOS1Mastership ) )
- main.log.warn( "ONOS2 mastership response: " +
- repr( ONOS2Mastership ) )
- main.log.warn( "ONOS3 mastership response: " +
- repr( ONOS3Mastership ) )
- main.log.warn( "ONOS4 mastership response: " +
- repr( ONOS4Mastership ) )
- main.log.warn( "ONOS5 mastership response: " +
- repr( ONOS5Mastership ) )
- main.log.warn( "ONOS6 mastership response: " +
- repr( ONOS6Mastership ) )
- main.log.warn( "ONOS7 mastership response: " +
- repr( ONOS7Mastership ) )
- consistentMastership = main.FALSE
- elif ONOS1Mastership == ONOS2Mastership\
- and ONOS1Mastership == ONOS3Mastership\
- and ONOS1Mastership == ONOS4Mastership\
- and ONOS1Mastership == ONOS5Mastership\
- and ONOS1Mastership == ONOS6Mastership\
- and ONOS1Mastership == ONOS7Mastership:
- consistentMastership = main.TRUE
+ ONOSMastership = []
+ mastershipCheck = main.FALSE
+ consistentMastership = True
+ rolesResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].roles,
+ name="roles-" + str( i ),
+ args=[] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSMastership.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " roles" )
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " mastership response: " +
+ repr( ONOSMastership[i] ) )
+ rolesResults = False
+ utilities.assert_equals(
+ expect=True,
+ actual=rolesResults,
+ onpass="No error in reading roles output",
+ onfail="Error in reading roles from ONOS" )
+
+ main.step( "Check for consistency in roles from each controller" )
+ if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
main.log.report(
"Switch roles are consistent across all ONOS nodes" )
else:
- main.log.warn( "ONOS1 roles: ", json.dumps(
- json.loads( ONOS1Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS2 roles: ", json.dumps(
- json.loads( ONOS2Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS3 roles: ", json.dumps(
- json.loads( ONOS3Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS4 roles: ", json.dumps(
- json.loads( ONOS4Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS5 roles: ", json.dumps(
- json.loads( ONOS5Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS6 roles: ", json.dumps(
- json.loads( ONOS6Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS7 roles: ", json.dumps(
- json.loads( ONOS7Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- consistentMastership = main.FALSE
+ consistentMastership = False
utilities.assert_equals(
- expect=main.TRUE,
+ expect=True,
actual=consistentMastership,
onpass="Switch roles are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of switch roles" )
+ if rolesResults and not consistentMastership:
+ for i in range( numControllers ):
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " roles: ",
+ json.dumps(
+ json.loads( ONOSMastership[ i ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ elif rolesResults and not consistentMastership:
+ mastershipCheck = main.TRUE
+
description2 = "Compare switch roles from before failure"
main.step( description2 )
-
- currentJson = json.loads( ONOS1Mastership )
- oldJson = json.loads( mastershipState )
+ try:
+ currentJson = json.loads( ONOSMastership[0] )
+ oldJson = json.loads( mastershipState )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Something is wrong with parsing " +
+ "ONOSMastership[0] or mastershipState" )
+ main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
+ main.log.error( "mastershipState" + repr( mastershipState ) )
+ main.cleanup()
+ main.exit()
mastershipCheck = main.TRUE
for i in range( 1, 29 ):
switchDPID = str(
- main.Mininet1.getSwitchDPID(
- switch="s" +
- str( i ) ) )
-
+ main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
current = [ switch[ 'master' ] for switch in currentJson
if switchDPID in switch[ 'id' ] ]
old = [ switch[ 'master' ] for switch in oldJson
@@ -1452,102 +1563,92 @@
mastershipCheck = consistentMastership
main.step( "Get the intents and compare across all nodes" )
- ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
- ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
- ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
- ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
- ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
- ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
- ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
+ ONOSIntents = []
intentCheck = main.FALSE
- if "Error" in ONOS1Intents or not ONOS1Intents\
- or "Error" in ONOS2Intents or not ONOS2Intents\
- or "Error" in ONOS3Intents or not ONOS3Intents\
- or "Error" in ONOS4Intents or not ONOS4Intents\
- or "Error" in ONOS5Intents or not ONOS5Intents\
- or "Error" in ONOS6Intents or not ONOS6Intents\
- or "Error" in ONOS7Intents or not ONOS7Intents:
- main.log.report( "Error in getting ONOS intents" )
- main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
- main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
- main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
- main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
- main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
- main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
- main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
- elif ONOS1Intents == ONOS2Intents\
- and ONOS1Intents == ONOS3Intents\
- and ONOS1Intents == ONOS4Intents\
- and ONOS1Intents == ONOS5Intents\
- and ONOS1Intents == ONOS6Intents\
- and ONOS1Intents == ONOS7Intents:
- intentCheck = main.TRUE
- main.log.report( "Intents are consistent across all ONOS nodes" )
- else:
- main.log.warn( "ONOS1 intents: " )
- print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS2 intents: " )
- print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS3 intents: " )
- print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS4 intents: " )
- print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS5 intents: " )
- print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS6 intents: " )
- print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS7 intents: " )
- print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
+ consistentIntents = True
+ intentsResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].intents,
+ name="intents-" + str( i ),
+ args=[],
+ kwargs={ 'jsonFormat': True } )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSIntents.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " intents" )
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
+ repr( ONOSIntents[ i ] ) )
+ intentsResults = False
utilities.assert_equals(
- expect=main.TRUE,
- actual=intentCheck,
+ expect=True,
+ actual=intentsResults,
+ onpass="No error in reading intents output",
+ onfail="Error in reading intents from ONOS" )
+
+ main.step( "Check for consistency in Intents from each controller" )
+ if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
+ main.log.report( "Intents are consistent across all ONOS " +
+ "nodes" )
+ else:
+ consistentIntents = False
+ utilities.assert_equals(
+ expect=True,
+ actual=consistentIntents,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
- # Print the intent states
- intents = []
- intents.append( ONOS1Intents )
- intents.append( ONOS2Intents )
- intents.append( ONOS3Intents )
- intents.append( ONOS4Intents )
- intents.append( ONOS5Intents )
- intents.append( ONOS6Intents )
- intents.append( ONOS7Intents )
intentStates = []
- for node in intents: # Iter through ONOS nodes
+ for node in ONOSIntents: # Iter through ONOS nodes
nodeStates = []
# Iter through intents of a node
- for intent in json.loads( node ):
- nodeStates.append( intent[ 'state' ] )
+ try:
+ for intent in json.loads( node ):
+ nodeStates.append( intent[ 'state' ] )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error in parsing intents" )
+ main.log.error( repr( node ) )
intentStates.append( nodeStates )
out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
main.log.info( dict( out ) )
+ if intentsResults and not consistentIntents:
+ for i in range( numControllers ):
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
+ main.log.warn( json.dumps(
+ json.loads( ONOSIntents[ i ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ elif intentsResults and consistentIntents:
+ intentCheck = main.TRUE
+
# NOTE: Store has no durability, so intents are lost across system
# restarts
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
sameIntents = main.TRUE
- if intentState and intentState == ONOS1Intents:
+ if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.report( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
- # what the aceptable states are
+ # what the acceptable states are
else:
try:
- main.log.warn( "ONOS1 intents: " )
- print json.dumps( json.loads( ONOS1Intents ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) )
- except Exception:
- pass
+ main.log.warn( "ONOS intents: " )
+ main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.warn( repr( ONOSIntents[0] ) )
sameIntents = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
@@ -1581,12 +1682,8 @@
main.step( "Check the continuous pings to ensure that no packets " +
"were dropped during component failure" )
- # FIXME: This check is always failing. Investigate cause
- # NOTE: this may be something to do with file permsissions
- # or slight change in format
- main.Mininet2.pingKill(
- main.params[ 'TESTONUSER' ],
- main.params[ 'TESTONIP' ] )
+ main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
+ main.params[ 'TESTONIP' ] )
LossInPings = main.FALSE
# NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range( 8, 18 ):
@@ -1611,11 +1708,13 @@
# Test of LeadershipElection
leaderList = []
+ # FIXME: make sure this matches nodes that were restarted
+ restarted = [ nodes[0].ip_address, nodes[1].ip_address,
+ nodes[2].ip_address ]
+
leaderResult = main.TRUE
- for controller in range( 1, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- leaderN = node.electionTestLeader()
+ for cli in CLIs:
+ leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
@@ -1624,14 +1723,12 @@
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.report( "ONOS" + str( controller ) +
+ main.log.report( cli.name +
" shows no leader for the election-app was" +
" elected after the old one died" )
leaderResult = main.FALSE
- elif leaderN == ONOS1Ip or leaderN == ONOS2Ip or\
- leaderN == ONOS3Ip:
- main.log.report( "ONOS" + str( controller ) +
- " shows " + str( leaderN ) +
+ elif leaderN in restarted:
+ main.log.report( cli.name + " shows " + str( leaderN ) +
" as leader for the election-app, but it " +
"was restarted" )
leaderResult = main.FALSE
@@ -1670,28 +1767,21 @@
from sts.topology.teston_topology import TestONTopology
import json
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
description = "Compare ONOS Topology view to Mininet topology"
main.case( description )
main.log.report( description )
main.step( "Create TestONTopology object" )
ctrls = []
- count = 1
- while True:
- temp = ()
- if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
- temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
- temp = temp + ( "ONOS" + str( count ), )
- temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
- temp = temp + \
- ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
- ctrls.append( temp )
- count = count + 1
- else:
- break
- MNTopo = TestONTopology(
- main.Mininet1,
- ctrls ) # can also add Intent API info for intent operations
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1705,28 +1795,40 @@
startTime = time.time()
# Give time for Gossip to work
while topoResult == main.FALSE and elapsed < 60:
- count = count + 1
+ count += 1
if count > 1:
- # TODO: Depricate STS usage
+ # TODO: Deprecate STS usage
MNTopo = TestONTopology( main.Mininet1, ctrls )
cliStart = time.time()
devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].devices,
+ name="devices-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ devices.append( t.result )
hosts = []
- hosts.append( json.loads( main.ONOScli1.hosts() ) )
- hosts.append( json.loads( main.ONOScli2.hosts() ) )
- hosts.append( json.loads( main.ONOScli3.hosts() ) )
- hosts.append( json.loads( main.ONOScli4.hosts() ) )
- hosts.append( json.loads( main.ONOScli5.hosts() ) )
- hosts.append( json.loads( main.ONOScli6.hosts() ) )
- hosts.append( json.loads( main.ONOScli7.hosts() ) )
ipResult = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].hosts,
+ name="hosts-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ try:
+ hosts.append( json.loads( t.result ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing hosts results" )
+ main.log.error( repr( t.result ) )
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
@@ -1736,29 +1838,41 @@
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].ports,
+ name="ports-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ports.append( t.result )
links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].links,
+ name="links-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ links.append( t.result )
clusters = []
- clusters.append( main.ONOScli1.clusters() )
- clusters.append( main.ONOScli2.clusters() )
- clusters.append( main.ONOScli3.clusters() )
- clusters.append( main.ONOScli4.clusters() )
- clusters.append( main.ONOScli5.clusters() )
- clusters.append( main.ONOScli6.clusters() )
- clusters.append( main.ONOScli7.clusters() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].clusters,
+ name="clusters-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ clusters.append( t.result )
elapsed = time.time() - startTime
cliTime = time.time() - cliStart
@@ -1877,7 +1991,11 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
- numClusters = len( json.loads( clusters[ 0 ] ) )
+ try:
+ numClusters = len( json.loads( clusters[ 0 ] ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing clusters[0]: " +
+ repr( clusters[0] ) )
clusterResults = main.FALSE
if numClusters == 1:
clusterResults = main.TRUE
@@ -1911,6 +2029,11 @@
Link s3-s28 down
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# NOTE: You should probably run a topology check after this
linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
@@ -1926,7 +2049,7 @@
" seconds for link down to be discovered" )
time.sleep( linkSleep )
utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
- onpass="Link down succesful",
+ onpass="Link down successful",
onfail="Failed to bring link down" )
# TODO do some sort of check here
@@ -1935,6 +2058,11 @@
Link s3-s28 up
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# NOTE: You should probably run a topology check after this
linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
@@ -1950,7 +2078,7 @@
" seconds for link up to be discovered" )
time.sleep( linkSleep )
utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
- onpass="Link up succesful",
+ onpass="Link up successful",
onfail="Failed to bring link up" )
# TODO do some sort of check here
@@ -1960,6 +2088,11 @@
"""
# NOTE: You should probably run a topology check after this
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
@@ -1983,7 +2116,7 @@
if device and device[ 'available' ] is False:
result = main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Kill switch succesful",
+ onpass="Kill switch successful",
onfail="Failed to kill switch?" )
def CASE12( self, main ):
@@ -1992,6 +2125,18 @@
"""
# NOTE: You should probably run a topology check after this
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert ONOS1Port, "ONOS1Port not defined"
+ assert ONOS2Port, "ONOS2Port not defined"
+ assert ONOS3Port, "ONOS3Port not defined"
+ assert ONOS4Port, "ONOS4Port not defined"
+ assert ONOS5Port, "ONOS5Port not defined"
+ assert ONOS6Port, "ONOS6Port not defined"
+ assert ONOS7Port, "ONOS7Port not defined"
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
switch = main.params[ 'kill' ][ 'switch' ]
@@ -2004,24 +2149,23 @@
main.step( "Add back " + switch )
main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
- # TODO: New dpid or same? Ask Thomas?
for peer in links:
main.Mininet1.addLink( switch, peer )
main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
count=numControllers,
- ip1=ONOS1Ip,
+ ip1=nodes[ 0 ].ip_address,
port1=ONOS1Port,
- ip2=ONOS2Ip,
+ ip2=nodes[ 1 ].ip_address,
port2=ONOS2Port,
- ip3=ONOS3Ip,
+ ip3=nodes[ 2 ].ip_address,
port3=ONOS3Port,
- ip4=ONOS4Ip,
+ ip4=nodes[ 3 ].ip_address,
port4=ONOS4Port,
- ip5=ONOS5Ip,
+ ip5=nodes[ 4 ].ip_address,
port5=ONOS5Port,
- ip6=ONOS6Ip,
+ ip6=nodes[ 5 ].ip_address,
port6=ONOS6Port,
- ip7=ONOS7Ip,
+ ip7=nodes[ 6 ].ip_address,
port7=ONOS7Port )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch up to be discovered" )
@@ -2033,7 +2177,7 @@
if device and device[ 'available' ]:
result = main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="add switch succesful",
+ onpass="add switch successful",
onfail="Failed to add switch?" )
def CASE13( self, main ):
@@ -2042,37 +2186,22 @@
"""
import os
import time
- # TODO: make use of this elsewhere
- ips = []
- ips.append( ONOS1Ip )
- ips.append( ONOS2Ip )
- ips.append( ONOS3Ip )
- ips.append( ONOS4Ip )
- ips.append( ONOS5Ip )
- ips.append( ONOS6Ip )
- ips.append( ONOS7Ip )
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# printing colors to terminal
- colors = {}
- colors[ 'cyan' ] = '\033[96m'
- colors[ 'purple' ] = '\033[95m'
- colors[ 'blue' ] = '\033[94m'
- colors[ 'green' ] = '\033[92m'
- colors[ 'yellow' ] = '\033[93m'
- colors[ 'red' ] = '\033[91m'
- colors[ 'end' ] = '\033[0m'
+ colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
+ 'blue': '\033[94m', 'green': '\033[92m',
+ 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
description = "Test Cleanup"
main.log.report( description )
main.case( description )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
- main.step( "Checking ONOS Logs for errors" )
- for i in range( 7 ):
- print colors[ 'purple' ] + "Checking logs for errors on " + \
- "ONOS" + str( i + 1 ) + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( ips[ i ] )
-
main.step( "Copying MN pcap and ONOS log files to test station" )
testname = main.TEST
teststationUser = main.params[ 'TESTONUSER' ]
@@ -2088,14 +2217,13 @@
# NOTE: must end in /
dstDir = "~/packet_captures/"
for f in logFiles:
- for i in range( 7 ):
- main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
- logFolder + f + " " +
+ for node in nodes:
+ main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
+ ":" + logFolder + f + " " +
teststationUser + "@" +
teststationIP + ":" +
dstDir + str( testname ) +
- "-ONOS" + str( i + 1 ) + "-" +
- f )
+ "-" + node.name + "-" + f )
main.ONOSbench.handle.expect( "\$" )
# std*.log's
@@ -2105,18 +2233,26 @@
# NOTE: must end in /
dstDir = "~/packet_captures/"
for f in logFiles:
- for i in range( 7 ):
- main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
- logFolder + f + " " +
+ for node in nodes:
+ main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
+ ":" + logFolder + f + " " +
teststationUser + "@" +
teststationIP + ":" +
dstDir + str( testname ) +
- "-ONOS" + str( i + 1 ) + "-" +
- f )
+ "-" + node.name + "-" + f )
main.ONOSbench.handle.expect( "\$" )
# sleep so scp can finish
time.sleep( 10 )
+
+ main.step( "Stopping Mininet" )
main.Mininet1.stopNet()
+
+ main.step( "Checking ONOS Logs for errors" )
+ for node in nodes:
+ print colors[ 'purple' ] + "Checking logs for errors on " + \
+ node.name + ":" + colors[ 'end' ]
+ print main.ONOSbench.checkLogs( node.ip_address )
+
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
@@ -2129,45 +2265,56 @@
"""
start election app on all onos nodes
"""
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
leaderResult = main.TRUE
# install app on onos 1
main.log.info( "Install leadership election app" )
main.ONOScli1.featureInstall( "onos-app-election" )
+ leader = nodes[0].ip_address
# wait for election
# check for leader
- leader = main.ONOScli1.electionTestLeader()
- # verify leader is ONOS1
- if leader == ONOS1Ip:
- # all is well
- pass
- elif leader is None:
- # No leader elected
- main.log.report( "No leader was elected" )
- leaderResult = main.FALSE
- elif leader == main.FALSE:
- # error in response
- # TODO: add check for "Command not found:" in the driver, this
- # means the app isn't loaded
- main.log.report( "Something is wrong with electionTestLeader" +
- " function, check the error logs" )
- leaderResult = main.FALSE
- else:
- # error in response
- main.log.report(
- "Unexpected response from electionTestLeader function:'" +
- str( leader ) +
- "'" )
- leaderResult = main.FALSE
-
- # install on other nodes and check for leader.
- # Should be onos1 and each app should show the same leader
- for controller in range( 2, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- node.featureInstall( "onos-app-election" )
- leaderN = node.electionTestLeader()
+ for i in range(2): # try this twice
+ leader1 = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
- if leaderN == ONOS1Ip:
+ if leader1 == leader:
+ # all is well
+ # pass
+ if i > 0:
+ main.log.warn( "It took ONOS sometime for leader to be" +
+ "elected" )
+ main.log.debug( "I don't think this should be happening" )
+ break
+ elif leader1 is None:
+ # No leader elected
+ main.log.report( "No leader was elected" )
+ leaderResult = main.FALSE
+ elif leader1 == main.FALSE:
+ # error in response
+ # TODO: add check for "Command not found:" in the driver, this
+ # means the app isn't loaded
+ main.log.report( "Something is wrong with electionTestLeader" +
+ " function, check the error logs" )
+ leaderResult = main.FALSE
+ else:
+ # error in response
+ main.log.report(
+ "Unexpected response from electionTestLeader function:'" +
+ str( leader1 ) + "'" )
+ leaderResult = main.FALSE
+ time.sleep(2)
+ # install on other nodes and check for leader.
+ # Leader should be ONOS1 and each app should show the same leader
+ for cli in CLIs[ 1: ]:
+ cli.featureInstall( "onos-app-election" )
+ leaderN = cli.electionTestLeader()
+ # verify leader is ONOS1
+ if leaderN == leader:
# all is well
pass
elif leaderN == main.FALSE:
@@ -2180,8 +2327,7 @@
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( "ONOS" + str( controller ) + " sees " +
- str( leaderN ) +
+ main.log.report( cli.name + " sees " + str( leaderN ) +
" as the leader of the election app. Leader" +
" should be " +
str( leader ) )
@@ -2199,6 +2345,12 @@
"""
Check that Leadership Election is still functional
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
main.log.report( description )
@@ -2207,28 +2359,18 @@
leader = main.ONOScli1.electionTestLeader()
# TODO: do some sanity checking on leader before using it
withdrawResult = main.FALSE
- if leader == ONOS1Ip:
- oldLeader = getattr( main, "ONOScli1" )
- elif leader == ONOS2Ip:
- oldLeader = getattr( main, "ONOScli2" )
- elif leader == ONOS3Ip:
- oldLeader = getattr( main, "ONOScli3" )
- elif leader == ONOS4Ip:
- oldLeader = getattr( main, "ONOScli4" )
- elif leader == ONOS5Ip:
- oldLeader = getattr( main, "ONOScli5" )
- elif leader == ONOS6Ip:
- oldLeader = getattr( main, "ONOScli6" )
- elif leader == ONOS7Ip:
- oldLeader = getattr( main, "ONOScli7" )
- elif leader is None or leader == main.FALSE:
+ if leader is None or leader == main.FALSE:
main.log.report(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
oldLeader = None
+ for i in range( len( CLIs ) ):
+ if leader == nodes[ i ].ip_address:
+ oldLeader = CLIs[ i ]
+ break
else:
- main.log.error( "Leader election --- why am I HERE?!?")
+ main.log.error( "Leader election, could not find current leader" )
if oldLeader:
withdrawResult = oldLeader.electionTestWithdraw()
utilities.assert_equals(
@@ -2239,21 +2381,17 @@
main.step( "Make sure new leader is elected" )
leaderList = []
- for controller in range( 1, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- leaderList.append( node.electionTestLeader() )
- for leaderN in leaderList:
+ for cli in CLIs:
+ leaderN = cli.electionTestLeader()
+ leaderList.append( leaderN )
if leaderN == leader:
- main.log.report(
- "ONOS" + str( controller ) +
- " still sees " + str( leader ) +
- " as leader after they withdrew" )
+ main.log.report( cli.name + " still sees " + str( leader ) +
+ " as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
- # means the app isn't loaded
+ # means the app isn't loaded
main.log.report( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
@@ -2270,6 +2408,7 @@
for n in range( len( leaderList ) ):
main.log.report( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
+ leaderResult = leaderResult and consistentLeader
if leaderResult:
main.log.report( "Leadership election tests passed( consistent " +
"view of leader across listeners and a new " +
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
index 99eb9e5..332dd47 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
@@ -529,7 +529,7 @@
if PingResult == main.FALSE:
main.log.report(
"Intents have not been installed correctly, pings failed." )
- main.log.warn( "ONSO1 intents: " )
+ main.log.warn( "ONOS1 intents: " )
main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
sort_keys=True,
indent=4,
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 9ba287d..5b3e125 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -49,7 +49,7 @@
main.case( "Setting up test environment" )
# TODO: save all the timers and output them for plotting
- # load some vairables from the params file
+ # load some variables from the params file
PULLCODE = False
if main.params[ 'Git' ] == 'True':
PULLCODE = True
@@ -57,38 +57,34 @@
cellName = main.params[ 'ENV' ][ 'cellName' ]
# set global variables
- global ONOS1Ip
global ONOS1Port
- global ONOS2Ip
global ONOS2Port
- global ONOS3Ip
global ONOS3Port
- global ONOS4Ip
global ONOS4Port
- global ONOS5Ip
global ONOS5Port
- global ONOS6Ip
global ONOS6Port
- global ONOS7Ip
global ONOS7Port
global numControllers
-
- ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
- ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
- ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
- ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
- ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
- ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
- ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
- ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
- ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
- ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
- ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
- ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
- ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
- ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
numControllers = int( main.params[ 'num_controllers' ] )
+ # FIXME: just get controller port from params?
+ # TODO: do we really need all these?
+ ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
+ ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
+ ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
+ ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
+ ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
+ ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
+ ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
+
+ global CLIs
+ CLIs = []
+ global nodes
+ nodes = []
+ for i in range( 1, numControllers + 1 ):
+ CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
+ nodes.append( getattr( main, 'ONOS' + str( i ) ) )
+
main.step( "Applying cell variable to environment" )
cellResult = main.ONOSbench.setCell( cellName )
verifyResult = main.ONOSbench.verifyCell()
@@ -96,14 +92,10 @@
# FIXME:this is short term fix
main.log.report( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
+
main.log.report( "Uninstalling ONOS" )
- main.ONOSbench.onosUninstall( ONOS1Ip )
- main.ONOSbench.onosUninstall( ONOS2Ip )
- main.ONOSbench.onosUninstall( ONOS3Ip )
- main.ONOSbench.onosUninstall( ONOS4Ip )
- main.ONOSbench.onosUninstall( ONOS5Ip )
- main.ONOSbench.onosUninstall( ONOS6Ip )
- main.ONOSbench.onosUninstall( ONOS7Ip )
+ for node in nodes:
+ main.ONOSbench.onosUninstall( node.ip_address )
cleanInstallResult = main.TRUE
gitPullResult = main.TRUE
@@ -116,6 +108,8 @@
main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
+ if gitPullResult == main.ERROR:
+ main.log.error( "Error pulling git branch" )
main.step( "Using mvn clean & install" )
cleanInstallResult = main.ONOSbench.cleanInstall()
@@ -128,76 +122,38 @@
packageResult = main.ONOSbench.onosPackage()
main.step( "Installing ONOS package" )
- onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS1Ip )
- onos2InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS2Ip )
- onos3InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS3Ip )
- onos4InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS4Ip )
- onos5InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS5Ip )
- onos6InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS6Ip )
- onos7InstallResult = main.ONOSbench.onosInstall( options="-f",
- node=ONOS7Ip )
- onosInstallResult = onos1InstallResult and onos2InstallResult\
- and onos3InstallResult and onos4InstallResult\
- and onos5InstallResult and onos6InstallResult\
- and onos7InstallResult
+ onosInstallResult = main.TRUE
+ for node in nodes:
+ tmpResult = main.ONOSbench.onosInstall( options="-f",
+ node=node.ip_address )
+ onosInstallResult = onosInstallResult and tmpResult
main.step( "Checking if ONOS is up yet" )
for i in range( 2 ):
- onos1Isup = main.ONOSbench.isup( ONOS1Ip )
- if not onos1Isup:
- main.log.report( "ONOS1 didn't start!" )
- main.ONOSbench.onosStop( ONOS1Ip )
- main.ONOSbench.onosStart( ONOS1Ip )
- onos2Isup = main.ONOSbench.isup( ONOS2Ip )
- if not onos2Isup:
- main.log.report( "ONOS2 didn't start!" )
- main.ONOSbench.onosStop( ONOS2Ip )
- main.ONOSbench.onosStart( ONOS2Ip )
- onos3Isup = main.ONOSbench.isup( ONOS3Ip )
- if not onos3Isup:
- main.log.report( "ONOS3 didn't start!" )
- main.ONOSbench.onosStop( ONOS3Ip )
- main.ONOSbench.onosStart( ONOS3Ip )
- onos4Isup = main.ONOSbench.isup( ONOS4Ip )
- if not onos4Isup:
- main.log.report( "ONOS4 didn't start!" )
- main.ONOSbench.onosStop( ONOS4Ip )
- main.ONOSbench.onosStart( ONOS4Ip )
- onos5Isup = main.ONOSbench.isup( ONOS5Ip )
- if not onos5Isup:
- main.log.report( "ONOS5 didn't start!" )
- main.ONOSbench.onosStop( ONOS5Ip )
- main.ONOSbench.onosStart( ONOS5Ip )
- onos6Isup = main.ONOSbench.isup( ONOS6Ip )
- if not onos6Isup:
- main.log.report( "ONOS6 didn't start!" )
- main.ONOSbench.onosStop( ONOS6Ip )
- main.ONOSbench.onosStart( ONOS6Ip )
- onos7Isup = main.ONOSbench.isup( ONOS7Ip )
- if not onos7Isup:
- main.log.report( "ONOS7 didn't start!" )
- main.ONOSbench.onosStop( ONOS7Ip )
- main.ONOSbench.onosStart( ONOS7Ip )
- onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
- and onos4Isup and onos5Isup and onos6Isup and onos7Isup
+ onosIsupResult = main.TRUE
+ for node in nodes:
+ started = main.ONOSbench.isup( node.ip_address )
+ if not started:
+ main.log.report( node.name + " didn't start!" )
+ main.ONOSbench.onosStop( node.ip_address )
+ main.ONOSbench.onosStart( node.ip_address )
+ onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
- cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
- cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
- cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
- cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
- cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
- cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
- cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
- cliResults = cliResult1 and cliResult2 and cliResult3 and\
- cliResult4 and cliResult5 and cliResult6 and cliResult7
+ main.log.step( "Starting ONOS CLI sessions" )
+ cliResults = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].startOnosCli,
+ name="startOnosCli-" + str( i ),
+ args=[nodes[i].ip_address] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ cliResults = cliResults and t.result
main.step( "Start Packet Capture MN" )
main.Mininet2.startTcpdump(
@@ -223,22 +179,36 @@
Assign mastership to controllers
"""
import re
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert ONOS1Port, "ONOS1Port not defined"
+ assert ONOS2Port, "ONOS2Port not defined"
+ assert ONOS3Port, "ONOS3Port not defined"
+ assert ONOS4Port, "ONOS4Port not defined"
+ assert ONOS5Port, "ONOS5Port not defined"
+ assert ONOS6Port, "ONOS6Port not defined"
+ assert ONOS7Port, "ONOS7Port not defined"
main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
main.step( "Assign switches to controllers" )
+ # TODO: rewrite this function to take lists of ips and ports?
+ # or list of tuples?
for i in range( 1, 29 ):
main.Mininet1.assignSwController(
sw=str( i ),
count=numControllers,
- ip1=ONOS1Ip, port1=ONOS1Port,
- ip2=ONOS2Ip, port2=ONOS2Port,
- ip3=ONOS3Ip, port3=ONOS3Port,
- ip4=ONOS4Ip, port4=ONOS4Port,
- ip5=ONOS5Ip, port5=ONOS5Port,
- ip6=ONOS6Ip, port6=ONOS6Port,
- ip7=ONOS7Ip, port7=ONOS7Port )
+ ip1=nodes[ 0 ].ip_address, port1=ONOS1Port,
+ ip2=nodes[ 1 ].ip_address, port2=ONOS2Port,
+ ip3=nodes[ 2 ].ip_address, port3=ONOS3Port,
+ ip4=nodes[ 3 ].ip_address, port4=ONOS4Port,
+ ip5=nodes[ 4 ].ip_address, port5=ONOS5Port,
+ ip6=nodes[ 5 ].ip_address, port6=ONOS6Port,
+ ip7=nodes[ 6 ].ip_address, port7=ONOS7Port )
mastershipCheck = main.TRUE
for i in range( 1, 29 ):
@@ -247,16 +217,11 @@
main.log.info( str( response ) )
except Exception:
main.log.info( repr( response ) )
- if re.search( "tcp:" + ONOS1Ip, response )\
- and re.search( "tcp:" + ONOS2Ip, response )\
- and re.search( "tcp:" + ONOS3Ip, response )\
- and re.search( "tcp:" + ONOS4Ip, response )\
- and re.search( "tcp:" + ONOS5Ip, response )\
- and re.search( "tcp:" + ONOS6Ip, response )\
- and re.search( "tcp:" + ONOS7Ip, response ):
- mastershipCheck = mastershipCheck and main.TRUE
- else:
- mastershipCheck = main.FALSE
+ for node in nodes:
+ if re.search( "tcp:" + node.ip_address, response ):
+ mastershipCheck = mastershipCheck and main.TRUE
+ else:
+ mastershipCheck = main.FALSE
if mastershipCheck == main.TRUE:
main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
@@ -264,19 +229,20 @@
actual=mastershipCheck,
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
-
+ #FIXME: turning off because of ONOS-1286
# Manually assign mastership to the controller we want
roleCall = main.TRUE
roleCheck = main.TRUE
try:
# Assign switch
+ ip = nodes[ 0 ].ip_address # ONOS1
deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
assert deviceId, "No device id for s1 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS1Ip )
+ ip )
# Check assignment
- if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
@@ -286,21 +252,22 @@
assert deviceId, "No device id for s28 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS1Ip )
+ ip )
# Check assignment
- if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 1 ].ip_address # ONOS2
# Assign switch
deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
assert deviceId, "No device id for s2 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS2Ip )
+ ip )
# Check assignment
- if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
@@ -310,21 +277,22 @@
assert deviceId, "No device id for s3 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS2Ip )
+ ip )
# Check assignment
- if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 2 ].ip_address # ONOS3
# Assign switch
deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
assert deviceId, "No device id for s5 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS3Ip )
+ ip )
# Check assignment
- if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
@@ -334,65 +302,68 @@
assert deviceId, "No device id for s6 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS3Ip )
+ ip )
# Check assignment
- if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 3 ].ip_address # ONOS4
# Assign switch
deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
assert deviceId, "No device id for s4 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS4Ip )
+ ip )
# Check assignment
- if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 4 ].ip_address # ONOS5
for i in range( 8, 18 ):
dpid = '3' + str( i ).zfill( 3 )
deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
assert deviceId, "No device id for s%i in ONOS" % i
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS5Ip )
+ ip )
# Check assignment
- if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 5 ].ip_address # ONOS6
deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
assert deviceId, "No device id for s7 in ONOS"
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS6Ip )
+ ip )
# Check assignment
- if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
+ ip = nodes[ 6 ].ip_address # ONOS7
for i in range( 18, 28 ):
dpid = '6' + str( i ).zfill( 3 )
deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
assert deviceId, "No device id for s%i in ONOS" % i
roleCall = roleCall and main.ONOScli1.deviceRole(
deviceId,
- ONOS7Ip )
+ ip )
# Check assignment
- if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
+ if ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
roleCheck = roleCheck and main.TRUE
else:
roleCheck = roleCheck and main.FALSE
except ( AttributeError, AssertionError ):
main.log.exception( "Something is wrong with ONOS device view" )
main.log.info( main.ONOScli1.devices() )
-
utilities.assert_equals(
expect=main.TRUE,
actual=roleCall,
@@ -417,6 +388,11 @@
"""
import time
import json
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
main.log.report( "Adding host intents" )
main.case( "Adding host Intents" )
@@ -425,39 +401,50 @@
# install onos-app-fwd
main.log.info( "Install reactive forwarding app" )
- main.ONOScli1.featureInstall( "onos-app-fwd" )
- main.ONOScli2.featureInstall( "onos-app-fwd" )
- main.ONOScli3.featureInstall( "onos-app-fwd" )
- main.ONOScli4.featureInstall( "onos-app-fwd" )
- main.ONOScli5.featureInstall( "onos-app-fwd" )
- main.ONOScli6.featureInstall( "onos-app-fwd" )
- main.ONOScli7.featureInstall( "onos-app-fwd" )
+ appResults = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].featureInstall,
+ name="featureInstall-" + str( i ),
+ args=["onos-app-fwd"] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ appResults = appResults and t.result
# REACTIVE FWD test
pingResult = main.FALSE
- time1 = time.time()
- pingResult = main.Mininet1.pingall()
- utilities.assert_equals(
- expect=main.TRUE,
- actual=pingResult,
- onpass="Reactive Pingall test passed",
- onfail="Reactive Pingall failed, one or more ping pairs failed" )
- time2 = time.time()
- main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
+ for i in range(2): # Retry if pingall fails first time
+ time1 = time.time()
+ pingResult = main.Mininet1.pingall()
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=pingResult,
+ onpass="Reactive Pingall test passed",
+ onfail="Reactive Pingall failed, one or more ping pairs failed" )
+ time2 = time.time()
+ main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
# uninstall onos-app-fwd
main.log.info( "Uninstall reactive forwarding app" )
- main.ONOScli1.featureUninstall( "onos-app-fwd" )
- main.ONOScli2.featureUninstall( "onos-app-fwd" )
- main.ONOScli3.featureUninstall( "onos-app-fwd" )
- main.ONOScli4.featureUninstall( "onos-app-fwd" )
- main.ONOScli5.featureUninstall( "onos-app-fwd" )
- main.ONOScli6.featureUninstall( "onos-app-fwd" )
- main.ONOScli7.featureUninstall( "onos-app-fwd" )
- # timeout for fwd flows
- time.sleep( 10 )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].featureUninstall,
+ name="featureUninstall-" + str( i ),
+ args=["onos-app-fwd"] )
+ threads.append( t )
+ t.start()
- main.step( "Add host intents" )
+ for t in threads:
+ t.join()
+ appResults = appResults and t.result
+
+ # timeout for fwd flows
+ time.sleep( 11 )
+
+ main.step( "Add host intents" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -479,43 +466,57 @@
host1Id = host1Dict.get( 'id', None )
host2Id = host2Dict.get( 'id', None )
if host1Id and host2Id:
- nodeNum = ( i % 7 ) + 1
- node = getattr( main, ( 'ONOScli' + str( nodeNum ) ) )
- tmpId = node.addHostIntent(
- host1Id,
- host2Id )
+ nodeNum = ( i % 7 )
+ tmpId = CLIs[ nodeNum ].addHostIntent( host1Id, host2Id )
if tmpId:
main.log.info( "Added intent with id: " + tmpId )
intentIds.append( tmpId )
else:
- main.log.error( "addHostIntent reutrned None" )
+ main.log.error( "addHostIntent returned: " +
+ repr( tmpId ) )
else:
- main.log.error( "Error, getHost() failed" )
- main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.error( "Error, getHost() failed for h" + str( i ) +
+ " and/or h" + str( i + 10 ) )
+ hosts = CLIs[ 0 ].hosts()
+ main.log.warn( "Hosts output: " )
+ try:
+ main.log.warn( json.dumps( json.loads( hosts ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( hosts ) )
hostResult = main.FALSE
+ # FIXME: DEBUG
+ intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
main.log.info( "Intents in ONOS: " + str( onosIds ) )
for intent in intentIds:
if intent in onosIds:
- pass # intent submitted is still in onos
+ pass # intent submitted is in onos
else:
intentAddResult = False
+ # FIXME: DEBUG
+ if intentAddResult:
+ intentStop = time.time()
+ else:
+ intentStop = None
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
- installedCheck = True
+ installedCheck = True
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
- for intent in json.loads( intents ): # Iter through intents of a node
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents" )
# add submitted intents not in the store
tmplist = [ i for i, s in intentStates ]
missingIntents = False
@@ -528,17 +529,63 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- # for node in nodes:
- # node.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: move this to the election test sections
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
pendingMap = main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
+
intentAddResult = bool( pingResult and hostResult and intentAddResult
and not missingIntents and installedCheck )
utilities.assert_equals(
@@ -546,6 +593,27 @@
actual=intentAddResult,
onpass="Pushed host intents to ONOS",
onfail="Error in pushing host intents to ONOS" )
+ for i in range(100):
+ onosIds = main.ONOScli1.getAllIntentsId()
+ main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
+ main.log.info( "Intents in ONOS: " + str( sorted( onosIds ) ) )
+ if sorted(onosIds) == sorted(intentIds):
+ break
+ else:
+ time.sleep(1)
+ # FIXME: DEBUG
+ if not intentStop:
+ intentStop = time.time()
+ gossipTime = intentStop - intentStart
+ main.log.info( "It took about " + str( gossipTime ) +
+ " seconds for all intents to appear on ONOS1" )
+ # FIXME: make this time configurable/calculate based off of # of nodes
+ # and gossip rounds
+ utilities.assert_greater_equals(
+ expect=30, actual=gossipTime,
+ onpass="ECM anti-entropy for intents worked within " +
+ "expected time",
+ onfail="Intent ECM anti-entropy took too long" )
if not intentAddResult or "key" in pendingMap:
import time
@@ -560,13 +628,16 @@
intentStates = []
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
- for intent in json.loads( intents ):
- # Iter through intents of a node
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ # Iter through intents of a node
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents" )
# add submitted intents not in the store
tmplist = [ i for i, s in intentStates ]
for i in intentIds:
@@ -577,20 +648,73 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
def CASE4( self, main ):
"""
Ping across added host intents
"""
import json
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
description = " Ping across added host intents"
main.log.report( description )
main.case( description )
@@ -609,11 +733,15 @@
main.log.report(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
- main.log.warn( "ONSO1 intents: " )
- main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
if PingResult == main.TRUE:
main.log.report(
"Intents have been installed correctly and verified by pings" )
@@ -631,28 +759,79 @@
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
# Iter through intents of a node
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
+
if not installedCheck:
- main.log.info( "Waiting 60 seconds to see if intent states change" )
+ main.log.info( "Waiting 60 seconds to see if the state of " +
+ "intents change" )
time.sleep( 60 )
# Print the intent states
intents = main.ONOScli1.intents()
@@ -660,32 +839,87 @@
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
# Iter through intents of a node
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- main.ONOScli1.pendingMap()
- main.ONOScli2.pendingMap()
- main.ONOScli3.pendingMap()
- main.ONOScli4.pendingMap()
- main.ONOScli5.pendingMap()
- main.ONOScli6.pendingMap()
- main.ONOScli7.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
def CASE5( self, main ):
"""
Reading state of ONOS
"""
import json
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
@@ -693,344 +927,232 @@
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
- # We can then compare them with eachother and also with past states
+ # We can then compare them with each other and also with past states
- main.step( "Get the Mastership of each switch from each controller" )
+ main.step( "Check that each switch has a master" )
global mastershipState
- mastershipState = []
+ mastershipState = '[]'
# Assert that each device has a master
- ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
- ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
- ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
- ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
- ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
- ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
- ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
- rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
- ONOS3MasterNotNull and ONOS4MasterNotNull and\
- ONOS5MasterNotNull and ONOS6MasterNotNull and\
- ONOS7MasterNotNull
+ rolesNotNull = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].rolesNotNull,
+ name="rolesNotNull-" + str( i ),
+ args=[] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ rolesNotNull = rolesNotNull and t.result
utilities.assert_equals(
expect=main.TRUE,
actual=rolesNotNull,
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
- ONOS1Mastership = main.ONOScli1.roles()
- ONOS2Mastership = main.ONOScli2.roles()
- ONOS3Mastership = main.ONOScli3.roles()
- ONOS4Mastership = main.ONOScli4.roles()
- ONOS5Mastership = main.ONOScli5.roles()
- ONOS6Mastership = main.ONOScli6.roles()
- ONOS7Mastership = main.ONOScli7.roles()
- if "Error" in ONOS1Mastership or not ONOS1Mastership\
- or "Error" in ONOS2Mastership or not ONOS2Mastership\
- or "Error" in ONOS3Mastership or not ONOS3Mastership\
- or "Error" in ONOS4Mastership or not ONOS4Mastership\
- or "Error" in ONOS5Mastership or not ONOS5Mastership\
- or "Error" in ONOS6Mastership or not ONOS6Mastership\
- or "Error" in ONOS7Mastership or not ONOS7Mastership:
- main.log.report( "Error in getting ONOS roles" )
- main.log.warn(
- "ONOS1 mastership response: " +
- repr( ONOS1Mastership ) )
- main.log.warn(
- "ONOS2 mastership response: " +
- repr( ONOS2Mastership ) )
- main.log.warn(
- "ONOS3 mastership response: " +
- repr( ONOS3Mastership ) )
- main.log.warn(
- "ONOS4 mastership response: " +
- repr( ONOS4Mastership ) )
- main.log.warn(
- "ONOS5 mastership response: " +
- repr( ONOS5Mastership ) )
- main.log.warn(
- "ONOS6 mastership response: " +
- repr( ONOS6Mastership ) )
- main.log.warn(
- "ONOS7 mastership response: " +
- repr( ONOS7Mastership ) )
- consistentMastership = main.FALSE
- elif ONOS1Mastership == ONOS2Mastership\
- and ONOS1Mastership == ONOS3Mastership\
- and ONOS1Mastership == ONOS4Mastership\
- and ONOS1Mastership == ONOS5Mastership\
- and ONOS1Mastership == ONOS6Mastership\
- and ONOS1Mastership == ONOS7Mastership:
- mastershipState = ONOS1Mastership
- consistentMastership = main.TRUE
+ main.step( "Get the Mastership of each switch from each controller" )
+ ONOSMastership = []
+ mastershipCheck = main.FALSE
+ consistentMastership = True
+ rolesResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].roles,
+ name="roles-" + str( i ),
+ args=[] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSMastership.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " roles" )
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " mastership response: " +
+ repr( ONOSMastership[i] ) )
+ rolesResults = False
+ utilities.assert_equals(
+ expect=True,
+ actual=rolesResults,
+ onpass="No error in reading roles output",
+ onfail="Error in reading roles from ONOS" )
+
+ main.step( "Check for consistency in roles from each controller" )
+ if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
main.log.report(
"Switch roles are consistent across all ONOS nodes" )
else:
- main.log.warn(
- "ONOS1 roles: ",
- json.dumps(
- json.loads( ONOS1Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS2 roles: ",
- json.dumps(
- json.loads( ONOS2Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS3 roles: ",
- json.dumps(
- json.loads( ONOS3Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS4 roles: ",
- json.dumps(
- json.loads( ONOS4Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS5 roles: ",
- json.dumps(
- json.loads( ONOS5Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS6 roles: ",
- json.dumps(
- json.loads( ONOS6Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS7 roles: ",
- json.dumps(
- json.loads( ONOS7Mastership ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- consistentMastership = main.FALSE
+ consistentMastership = False
utilities.assert_equals(
- expect=main.TRUE,
+ expect=True,
actual=consistentMastership,
onpass="Switch roles are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of switch roles" )
+ if rolesResults and not consistentMastership:
+ for i in range( numControllers ):
+ try:
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " roles: ",
+ json.dumps(
+ json.loads( ONOSMastership[ i ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( ONOSMastership[ i ] ) )
+ elif rolesResults and consistentMastership:
+ mastershipCheck = main.TRUE
+ mastershipState = ONOSMastership[ 0 ]
+
main.step( "Get the intents from each controller" )
global intentState
intentState = []
- ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
- ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
- ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
- ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
- ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
- ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
- ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
+ ONOSIntents = []
intentCheck = main.FALSE
- if "Error" in ONOS1Intents or not ONOS1Intents\
- or "Error" in ONOS2Intents or not ONOS2Intents\
- or "Error" in ONOS3Intents or not ONOS3Intents\
- or "Error" in ONOS4Intents or not ONOS4Intents\
- or "Error" in ONOS5Intents or not ONOS5Intents\
- or "Error" in ONOS6Intents or not ONOS6Intents\
- or "Error" in ONOS7Intents or not ONOS7Intents:
- main.log.report( "Error in getting ONOS intents" )
- main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
- main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
- main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
- main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
- main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
- main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
- main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
- elif ONOS1Intents == ONOS2Intents\
- and ONOS1Intents == ONOS3Intents\
- and ONOS1Intents == ONOS4Intents\
- and ONOS1Intents == ONOS5Intents\
- and ONOS1Intents == ONOS6Intents\
- and ONOS1Intents == ONOS7Intents:
- intentState = ONOS1Intents
- intentCheck = main.TRUE
- main.log.report( "Intents are consistent across all ONOS nodes" )
- else:
- main.log.warn(
- "ONOS1 intents: ",
- json.dumps(
- json.loads( ONOS1Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS2 intents: ",
- json.dumps(
- json.loads( ONOS2Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS3 intents: ",
- json.dumps(
- json.loads( ONOS3Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS4 intents: ",
- json.dumps(
- json.loads( ONOS4Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS5 intents: ",
- json.dumps(
- json.loads( ONOS5Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS6 intents: ",
- json.dumps(
- json.loads( ONOS6Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
- main.log.warn(
- "ONOS7 intents: ",
- json.dumps(
- json.loads( ONOS7Intents ),
- sort_keys=True,
- indent=4,
- separators=(
- ',',
- ': ' ) ) )
+ consistentIntents = True
+ intentsResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].intents,
+ name="intents-" + str( i ),
+ args=[],
+ kwargs={ 'jsonFormat': True } )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSIntents.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " intents" )
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
+ repr( ONOSIntents[ i ] ) )
+ intentsResults = False
utilities.assert_equals(
- expect=main.TRUE,
- actual=intentCheck,
+ expect=True,
+ actual=intentsResults,
+ onpass="No error in reading intents output",
+ onfail="Error in reading intents from ONOS" )
+
+ main.step( "Check for consistency in Intents from each controller" )
+ if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
+ main.log.report( "Intents are consistent across all ONOS " +
+ "nodes" )
+ else:
+ consistentIntents = False
+ main.log.report( "Intents not consistent" )
+ utilities.assert_equals(
+ expect=True,
+ actual=consistentIntents,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
+ if intentsResults and not consistentIntents:
+ n = len(ONOSIntents)
+ main.log.warn( "ONOS" + str( n ) + " intents: " )
+ main.log.warn( json.dumps( json.loads( ONOSIntents[ -1 ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ for i in range( numControllers ):
+ if ONOSIntents[ i ] != ONOSIntents[ -1 ]:
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
+ main.log.warn( json.dumps( json.loads( ONOSIntents[i] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ else:
+ main.log.warn( nodes[ i ].name + " intents match ONOS" +
+ str( n ) + " intents" )
+ elif intentsResults and consistentIntents:
+ intentCheck = main.TRUE
+ intentState = ONOSIntents[ 0 ]
+
main.step( "Get the flows from each controller" )
global flowState
flowState = []
+ ONOSFlows = []
+ ONOSFlowsJson = []
flowCheck = main.FALSE
- try:
- ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
- ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
- ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
- ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
- ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
- ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
- ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
- assert ONOS1Flows, "ONOS1 Flows should not be empty"
- assert ONOS2Flows, "ONOS2 Flows should not be empty"
- assert ONOS3Flows, "ONOS3 Flows should not be empty"
- assert ONOS4Flows, "ONOS4 Flows should not be empty"
- assert ONOS5Flows, "ONOS5 Flows should not be empty"
- assert ONOS6Flows, "ONOS6 Flows should not be empty"
- assert ONOS7Flows, "ONOS7 Flows should not be empty"
- assert "Error" not in ONOS1Flows, "ONOS1 Flows contains 'Error'"
- assert "Error" not in ONOS2Flows, "ONOS2 Flows contains 'Error'"
- assert "Error" not in ONOS3Flows, "ONOS3 Flows contains 'Error'"
- assert "Error" not in ONOS4Flows, "ONOS4 Flows contains 'Error'"
- assert "Error" not in ONOS5Flows, "ONOS5 Flows contains 'Error'"
- assert "Error" not in ONOS6Flows, "ONOS6 Flows contains 'Error'"
- assert "Error" not in ONOS7Flows, "ONOS7 Flows contains 'Error'"
- ONOS1FlowsJson = json.loads( ONOS1Flows )
- ONOS2FlowsJson = json.loads( ONOS2Flows )
- ONOS3FlowsJson = json.loads( ONOS3Flows )
- ONOS4FlowsJson = json.loads( ONOS4Flows )
- ONOS5FlowsJson = json.loads( ONOS5Flows )
- ONOS6FlowsJson = json.loads( ONOS6Flows )
- ONOS7FlowsJson = json.loads( ONOS7Flows )
- except ( ValueError, AssertionError ): # From json.loads, or asserts
- main.log.exception( "One or more 'flows' responses from " +
- "ONOS couldn't be decoded." )
- main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
- main.log.warn( "ONOS2 flows repsponse: " + ONOS2Flows )
- main.log.warn( "ONOS3 flows repsponse: " + ONOS3Flows )
- main.log.warn( "ONOS4 flows repsponse: " + ONOS4Flows )
- main.log.warn( "ONOS5 flows repsponse: " + ONOS5Flows )
- main.log.warn( "ONOS6 flows repsponse: " + ONOS6Flows )
- main.log.warn( "ONOS7 flows repsponse: " + ONOS7Flows )
- else: # No exceptions
- if len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
- and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
- # TODO: Do a better check, maybe compare flows on switches?
- # NOTE Possible issue with this not always being set?
- flowState = ONOS1Flows
- flowCheck = main.TRUE
- main.log.report( "Flow count is consistent across all" +
- " ONOS nodes" )
+ consistentFlows = True
+ flowsResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].flows,
+ name="flows-" + str( i ),
+ args=[],
+ kwargs={ 'jsonFormat': True } )
+ threads.append( t )
+ t.start()
+
+ # FIXME: why am I sleeping here?
+ time.sleep(30)
+ for t in threads:
+ t.join()
+ result = t.result
+ ONOSFlows.append( result )
+
+ for i in range( numControllers ):
+ num = str( i + 1 )
+ if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
+ main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.warn( "ONOS" + num + " flows response: " +
+ repr( ONOSFlows[ i ] ) )
+ flowsResults = False
+ ONOSFlowsJson.append( None )
else:
- main.log.warn( "ONOS1 flows: " +
- json.dumps( ONOS1FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS2 flows: " +
- json.dumps( ONOS2FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS3 flows: " +
- json.dumps( ONOS3FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS4 flows: " +
- json.dumps( ONOS4FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS5 flows: " +
- json.dumps( ONOS5FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS6 flows: " +
- json.dumps( ONOS6FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS7 flows: " +
- json.dumps( ONOS7FlowsJson, sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ try:
+ ONOSFlowsJson.append( json.loads( ONOSFlows[ i ] ) )
+ except ( ValueError, TypeError ):
+ # FIXME: change this to log.error?
+ main.log.exception( "Error in parsing ONOS" + num +
+ " response as json." )
+ main.log.error( repr( ONOSFlows[ i ] ) )
+ ONOSFlowsJson.append( None )
+ flowsResults = False
utilities.assert_equals(
- expect=main.TRUE,
- actual=flowCheck,
+ expect=True,
+ actual=flowsResults,
+ onpass="No error in reading flows output",
+ onfail="Error in reading flows from ONOS" )
+
+ main.step( "Check for consistency in Flows from each controller" )
+ tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
+ if all( tmp ):
+ main.log.report( "Flow count is consistent across all ONOS nodes" )
+ else:
+ consistentFlows = False
+ utilities.assert_equals(
+ expect=True,
+ actual=consistentFlows,
onpass="The flow count is consistent across all ONOS nodes",
onfail="ONOS nodes have different flow counts" )
+ if flowsResults and not consistentFlows:
+ for i in range( numControllers ):
+ try:
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " flows: " +
+ json.dumps( json.loads( ONOSFlows[i] ), sort_keys=True,
+ indent=4, separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " flows: " +
+ repr( ONOSFlows[ i ] ) )
+ elif flowsResults and consistentFlows:
+ flowCheck = main.TRUE
+ flowState = ONOSFlows[ 0 ]
+
main.step( "Get the OF Table entries" )
global flows
flows = []
@@ -1085,64 +1207,80 @@
main.step( "Create TestONTopology object" )
ctrls = []
- count = 1
- while True:
- temp = ()
- if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
- temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
- temp = temp + ( "ONOS" + str( count ), )
- temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
- temp = temp + \
- ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
- ctrls.append( temp )
- count = count + 1
- else:
- break
- MNTopo = TestONTopology(
- main.Mininet1,
- ctrls ) # can also add Intent API info for intent operations
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
main.step( "Collecting topology information from ONOS" )
devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].devices,
+ name="devices-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ devices.append( t.result )
hosts = []
- hosts.append( json.loads( main.ONOScli1.hosts() ) )
- hosts.append( json.loads( main.ONOScli2.hosts() ) )
- hosts.append( json.loads( main.ONOScli3.hosts() ) )
- hosts.append( json.loads( main.ONOScli4.hosts() ) )
- hosts.append( json.loads( main.ONOScli5.hosts() ) )
- hosts.append( json.loads( main.ONOScli6.hosts() ) )
- hosts.append( json.loads( main.ONOScli7.hosts() ) )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].hosts,
+ name="hosts-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ try:
+ hosts.append( json.loads( t.result ) )
+ except ( ValueError, TypeError ):
+ # FIXME: better handling of this, print which node
+ # Maybe use thread name?
+ main.log.exception( "Error parsing json output of hosts" )
+ # FIXME: should this be an empty json object instead?
+ hosts.append( None )
+
ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].ports,
+ name="ports-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ports.append( t.result )
links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].links,
+ name="links-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ links.append( t.result )
clusters = []
- clusters.append( main.ONOScli1.clusters() )
- clusters.append( main.ONOScli2.clusters() )
- clusters.append( main.ONOScli3.clusters() )
- clusters.append( main.ONOScli4.clusters() )
- clusters.append( main.ONOScli5.clusters() )
- clusters.append( main.ONOScli6.clusters() )
- clusters.append( main.ONOScli7.clusters() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].clusters,
+ name="clusters-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ clusters.append( t.result )
# Compare json objects for hosts and dataplane clusters
# hosts
@@ -1176,10 +1314,9 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host.get( 'ips', [] ) == []:
- main.log.error(
- "DEBUG:Error with host ips on controller" +
- controllerStr + ": " + str( host ) )
+ if not host.get( 'ips', [ ] ):
+ main.log.error( "DEBUG:Error with host ips on controller" +
+ controllerStr + ": " + str( host ) )
ipResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
@@ -1190,12 +1327,12 @@
# Strongly connected clusters of devices
consistentClustersResult = main.TRUE
for controller in range( len( clusters ) ):
+ controllerStr = str( controller + 1 )
if "Error" not in clusters[ controller ]:
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
- controllerStr +
+ main.log.report( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
@@ -1212,7 +1349,11 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
- numClusters = len( json.loads( clusters[ 0 ] ) )
+ try:
+ numClusters = len( json.loads( clusters[ 0 ] ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing clusters[0]: " +
+ repr( clusters[ 0 ] ) )
clusterResults = main.FALSE
if numClusters == 1:
clusterResults = main.TRUE
@@ -1231,8 +1372,7 @@
if devices[ controller ] or "Error" not in devices[ controller ]:
currentDevicesResult = main.Mininet1.compareSwitches(
MNTopo,
- json.loads(
- devices[ controller ] ) )
+ json.loads( devices[ controller ] ) )
else:
currentDevicesResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
@@ -1245,8 +1385,7 @@
if ports[ controller ] or "Error" not in ports[ controller ]:
currentPortsResult = main.Mininet1.comparePorts(
MNTopo,
- json.loads(
- ports[ controller ] ) )
+ json.loads( ports[ controller ] ) )
else:
currentPortsResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
@@ -1259,8 +1398,7 @@
if links[ controller ] or "Error" not in links[ controller ]:
currentLinksResult = main.Mininet1.compareLinks(
MNTopo,
- json.loads(
- links[ controller ] ) )
+ json.loads( links[ controller ] ) )
else:
currentLinksResult = main.FALSE
utilities.assert_equals( expect=main.TRUE,
@@ -1274,16 +1412,17 @@
portsResults = portsResults and currentPortsResult
linksResults = linksResults and currentLinksResult
- topoResult = devicesResults and portsResults and linksResults\
- and consistentHostsResult and consistentClustersResult\
- and clusterResults and ipResult
+ topoResult = ( devicesResults and portsResults and linksResults
+ and consistentHostsResult and consistentClustersResult
+ and clusterResults and ipResult )
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
finalAssert = main.TRUE
- finalAssert = finalAssert and topoResult and flowCheck \
- and intentCheck and consistentMastership and rolesNotNull
+ finalAssert = ( finalAssert and topoResult and flowCheck
+ and intentCheck and consistentMastership
+ and rolesNotNull )
utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
onpass="State check successful",
onfail="State check NOT successful" )
@@ -1293,6 +1432,11 @@
The Failure case. Since this is the Sanity test, we do nothing.
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
main.log.report( "Wait 60 seconds instead of inducing a failure" )
time.sleep( 60 )
utilities.assert_equals(
@@ -1306,107 +1450,103 @@
Check state after ONOS failure
"""
import json
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
main.case( "Running ONOS Constant State Tests" )
+ main.step( "Check that each switch has a master" )
# Assert that each device has a master
- ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
- ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
- ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
- ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
- ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
- ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
- ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
- rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
- ONOS3MasterNotNull and ONOS4MasterNotNull and\
- ONOS5MasterNotNull and ONOS6MasterNotNull and\
- ONOS7MasterNotNull
+ rolesNotNull = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].rolesNotNull,
+ name="rolesNotNull-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ rolesNotNull = rolesNotNull and t.result
utilities.assert_equals(
expect=main.TRUE,
actual=rolesNotNull,
onpass="Each device has a master",
onfail="Some devices don't have a master assigned" )
- main.step( "Check if switch roles are consistent across all nodes" )
- ONOS1Mastership = main.ONOScli1.roles()
- ONOS2Mastership = main.ONOScli2.roles()
- ONOS3Mastership = main.ONOScli3.roles()
- ONOS4Mastership = main.ONOScli4.roles()
- ONOS5Mastership = main.ONOScli5.roles()
- ONOS6Mastership = main.ONOScli6.roles()
- ONOS7Mastership = main.ONOScli7.roles()
- if "Error" in ONOS1Mastership or not ONOS1Mastership\
- or "Error" in ONOS2Mastership or not ONOS2Mastership\
- or "Error" in ONOS3Mastership or not ONOS3Mastership\
- or "Error" in ONOS4Mastership or not ONOS4Mastership\
- or "Error" in ONOS5Mastership or not ONOS5Mastership\
- or "Error" in ONOS6Mastership or not ONOS6Mastership\
- or "Error" in ONOS7Mastership or not ONOS7Mastership:
- main.log.error( "Error in getting ONOS mastership" )
- main.log.warn( "ONOS1 mastership response: " +
- repr( ONOS1Mastership ) )
- main.log.warn( "ONOS2 mastership response: " +
- repr( ONOS2Mastership ) )
- main.log.warn( "ONOS3 mastership response: " +
- repr( ONOS3Mastership ) )
- main.log.warn( "ONOS4 mastership response: " +
- repr( ONOS4Mastership ) )
- main.log.warn( "ONOS5 mastership response: " +
- repr( ONOS5Mastership ) )
- main.log.warn( "ONOS6 mastership response: " +
- repr( ONOS6Mastership ) )
- main.log.warn( "ONOS7 mastership response: " +
- repr( ONOS7Mastership ) )
- consistentMastership = main.FALSE
- elif ONOS1Mastership == ONOS2Mastership\
- and ONOS1Mastership == ONOS3Mastership\
- and ONOS1Mastership == ONOS4Mastership\
- and ONOS1Mastership == ONOS5Mastership\
- and ONOS1Mastership == ONOS6Mastership\
- and ONOS1Mastership == ONOS7Mastership:
- consistentMastership = main.TRUE
+ ONOSMastership = []
+ mastershipCheck = main.FALSE
+ consistentMastership = True
+ rolesResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].roles,
+ name="roles-" + str( i ),
+ args=[] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSMastership.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " roles" )
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " mastership response: " +
+ repr( ONOSMastership[i] ) )
+ rolesResults = False
+ utilities.assert_equals(
+ expect=True,
+ actual=rolesResults,
+ onpass="No error in reading roles output",
+ onfail="Error in reading roles from ONOS" )
+
+ main.step( "Check for consistency in roles from each controller" )
+ if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
main.log.report(
"Switch roles are consistent across all ONOS nodes" )
else:
- main.log.warn( "ONOS1 roles: ", json.dumps(
- json.loads( ONOS1Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS2 roles: ", json.dumps(
- json.loads( ONOS2Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS3 roles: ", json.dumps(
- json.loads( ONOS3Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS4 roles: ", json.dumps(
- json.loads( ONOS4Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS5 roles: ", json.dumps(
- json.loads( ONOS5Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS6 roles: ", json.dumps(
- json.loads( ONOS6Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- main.log.warn( "ONOS7 roles: ", json.dumps(
- json.loads( ONOS7Mastership ), sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
- consistentMastership = main.FALSE
+ consistentMastership = False
utilities.assert_equals(
- expect=main.TRUE,
+ expect=True,
actual=consistentMastership,
onpass="Switch roles are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of switch roles" )
+ if rolesResults and not consistentMastership:
+ for i in range( numControllers ):
+ main.log.warn(
+ "ONOS" + str( i + 1 ) + " roles: ",
+ json.dumps(
+ json.loads( ONOSMastership[ i ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ elif rolesResults and not consistentMastership:
+ mastershipCheck = main.TRUE
+
description2 = "Compare switch roles from before failure"
main.step( description2 )
-
- currentJson = json.loads( ONOS1Mastership )
- oldJson = json.loads( mastershipState )
+ try:
+ currentJson = json.loads( ONOSMastership[0] )
+ oldJson = json.loads( mastershipState )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Something is wrong with parsing " +
+ "ONOSMastership[0] or mastershipState" )
+ main.log.error( "ONOSMastership[0]: " + repr( ONOSMastership[0] ) )
+ main.log.error( "mastershipState" + repr( mastershipState ) )
+ main.cleanup()
+ main.exit()
mastershipCheck = main.TRUE
for i in range( 1, 29 ):
switchDPID = str(
- main.Mininet1.getSwitchDPID(
- switch="s" +
- str( i ) ) )
-
+ main.Mininet1.getSwitchDPID( switch="s" + str( i ) ) )
current = [ switch[ 'master' ] for switch in currentJson
if switchDPID in switch[ 'id' ] ]
old = [ switch[ 'master' ] for switch in oldJson
@@ -1426,102 +1566,92 @@
mastershipCheck = mastershipCheck and consistentMastership
main.step( "Get the intents and compare across all nodes" )
- ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
- ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
- ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
- ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
- ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
- ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
- ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
+ ONOSIntents = []
intentCheck = main.FALSE
- if "Error" in ONOS1Intents or not ONOS1Intents\
- or "Error" in ONOS2Intents or not ONOS2Intents\
- or "Error" in ONOS3Intents or not ONOS3Intents\
- or "Error" in ONOS4Intents or not ONOS4Intents\
- or "Error" in ONOS5Intents or not ONOS5Intents\
- or "Error" in ONOS6Intents or not ONOS6Intents\
- or "Error" in ONOS7Intents or not ONOS7Intents:
- main.log.report( "Error in getting ONOS intents" )
- main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
- main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
- main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
- main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
- main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
- main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
- main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
- elif ONOS1Intents == ONOS2Intents\
- and ONOS1Intents == ONOS3Intents\
- and ONOS1Intents == ONOS4Intents\
- and ONOS1Intents == ONOS5Intents\
- and ONOS1Intents == ONOS6Intents\
- and ONOS1Intents == ONOS7Intents:
- intentCheck = main.TRUE
- main.log.report( "Intents are consistent across all ONOS nodes" )
- else:
- main.log.warn( "ONOS1 intents: " )
- print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS2 intents: " )
- print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS3 intents: " )
- print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS4 intents: " )
- print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS5 intents: " )
- print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS6 intents: " )
- print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
- main.log.warn( "ONOS7 intents: " )
- print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
- indent=4, separators=( ',', ': ' ) )
+ consistentIntents = True
+ intentsResults = True
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].intents,
+ name="intents-" + str( i ),
+ args=[],
+ kwargs={ 'jsonFormat': True } )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ONOSIntents.append( t.result )
+
+ for i in range( numControllers ):
+ if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
+ main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ " intents" )
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
+ repr( ONOSIntents[ i ] ) )
+ intentsResults = False
utilities.assert_equals(
- expect=main.TRUE,
- actual=intentCheck,
+ expect=True,
+ actual=intentsResults,
+ onpass="No error in reading intents output",
+ onfail="Error in reading intents from ONOS" )
+
+ main.step( "Check for consistency in Intents from each controller" )
+ if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
+ main.log.report( "Intents are consistent across all ONOS " +
+ "nodes" )
+ else:
+ consistentIntents = False
+ utilities.assert_equals(
+ expect=True,
+ actual=consistentIntents,
onpass="Intents are consistent across all ONOS nodes",
onfail="ONOS nodes have different views of intents" )
- # Print the intent states
- intents = []
- intents.append( ONOS1Intents )
- intents.append( ONOS2Intents )
- intents.append( ONOS3Intents )
- intents.append( ONOS4Intents )
- intents.append( ONOS5Intents )
- intents.append( ONOS6Intents )
- intents.append( ONOS7Intents )
intentStates = []
- for node in intents: # Iter through ONOS nodes
+ for node in ONOSIntents: # Iter through ONOS nodes
nodeStates = []
# Iter through intents of a node
- for intent in json.loads( node ):
- nodeStates.append( intent[ 'state' ] )
+ try:
+ for intent in json.loads( node ):
+ nodeStates.append( intent[ 'state' ] )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error in parsing intents" )
+ main.log.error( repr( node ) )
intentStates.append( nodeStates )
out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
main.log.info( dict( out ) )
+ if intentsResults and not consistentIntents:
+ for i in range( numControllers ):
+ main.log.warn( "ONOS" + str( i + 1 ) + " intents: " )
+ main.log.warn( json.dumps(
+ json.loads( ONOSIntents[ i ] ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ elif intentsResults and consistentIntents:
+ intentCheck = main.TRUE
+
# NOTE: Store has no durability, so intents are lost across system
# restarts
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
sameIntents = main.TRUE
- if intentState and intentState == ONOS1Intents:
+ if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.report( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
- # what the aceptable states are
+ # what the acceptable states are
else:
try:
- main.log.warn( "ONOS1 intents: " )
- print json.dumps( json.loads( ONOS1Intents ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) )
- except Exception:
- pass
+ main.log.warn( "ONOS intents: " )
+ main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.warn( repr( ONOSIntents[0] ) )
sameIntents = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
@@ -1555,12 +1685,8 @@
main.step( "Check the continuous pings to ensure that no packets " +
"were dropped during component failure" )
- # FIXME: This check is always failing. Investigate cause
- # NOTE: this may be something to do with file permsissions
- # or slight change in format
- main.Mininet2.pingKill(
- main.params[ 'TESTONUSER' ],
- main.params[ 'TESTONIP' ] )
+ main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
+ main.params[ 'TESTONIP' ] )
LossInPings = main.FALSE
# NOTE: checkForLoss returns main.FALSE with 0% packet loss
for i in range( 8, 18 ):
@@ -1586,12 +1712,10 @@
# Test of LeadershipElection
# NOTE: this only works for the sanity test. In case of failures,
# leader will likely change
- leader = ONOS1Ip
+ leader = nodes[ 0 ].ip_address
leaderResult = main.TRUE
- for controller in range( 1, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- leaderN = node.electionTestLeader()
+ for cli in CLIs:
+ leaderN = cli.electionTestLeader()
# verify leader is ONOS1
if leaderN == leader:
# all is well
@@ -1599,15 +1723,14 @@
# check != ONOS1
pass
elif leaderN == main.FALSE:
- # error in response
+ # error in response
main.log.report( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( "ONOS" + str( controller ) + " sees " +
- str( leaderN ) +
+ main.log.report( cli.name + " sees " + str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
if leaderResult:
@@ -1640,28 +1763,21 @@
from sts.topology.teston_topology import TestONTopology
import json
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
description = "Compare ONOS Topology view to Mininet topology"
main.case( description )
main.log.report( description )
main.step( "Create TestONTopology object" )
ctrls = []
- count = 1
- while True:
- temp = ()
- if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
- temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
- temp = temp + ( "ONOS" + str( count ), )
- temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
- temp = temp + \
- ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
- ctrls.append( temp )
- count = count + 1
- else:
- break
- MNTopo = TestONTopology(
- main.Mininet1,
- ctrls ) # can also add Intent API info for intent operations
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1675,28 +1791,40 @@
startTime = time.time()
# Give time for Gossip to work
while topoResult == main.FALSE and elapsed < 60:
- count = count + 1
+ count += 1
if count > 1:
- # TODO: Depricate STS usage
+ # TODO: Deprecate STS usage
MNTopo = TestONTopology( main.Mininet1, ctrls )
cliStart = time.time()
devices = []
- devices.append( main.ONOScli1.devices() )
- devices.append( main.ONOScli2.devices() )
- devices.append( main.ONOScli3.devices() )
- devices.append( main.ONOScli4.devices() )
- devices.append( main.ONOScli5.devices() )
- devices.append( main.ONOScli6.devices() )
- devices.append( main.ONOScli7.devices() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].devices,
+ name="devices-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ devices.append( t.result )
hosts = []
- hosts.append( json.loads( main.ONOScli1.hosts() ) )
- hosts.append( json.loads( main.ONOScli2.hosts() ) )
- hosts.append( json.loads( main.ONOScli3.hosts() ) )
- hosts.append( json.loads( main.ONOScli4.hosts() ) )
- hosts.append( json.loads( main.ONOScli5.hosts() ) )
- hosts.append( json.loads( main.ONOScli6.hosts() ) )
- hosts.append( json.loads( main.ONOScli7.hosts() ) )
ipResult = main.TRUE
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].hosts,
+ name="hosts-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ try:
+ hosts.append( json.loads( t.result ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing hosts results" )
+ main.log.error( repr( t.result ) )
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
@@ -1706,29 +1834,41 @@
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
- ports.append( main.ONOScli1.ports() )
- ports.append( main.ONOScli2.ports() )
- ports.append( main.ONOScli3.ports() )
- ports.append( main.ONOScli4.ports() )
- ports.append( main.ONOScli5.ports() )
- ports.append( main.ONOScli6.ports() )
- ports.append( main.ONOScli7.ports() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].ports,
+ name="ports-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ ports.append( t.result )
links = []
- links.append( main.ONOScli1.links() )
- links.append( main.ONOScli2.links() )
- links.append( main.ONOScli3.links() )
- links.append( main.ONOScli4.links() )
- links.append( main.ONOScli5.links() )
- links.append( main.ONOScli6.links() )
- links.append( main.ONOScli7.links() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].links,
+ name="links-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ links.append( t.result )
clusters = []
- clusters.append( main.ONOScli1.clusters() )
- clusters.append( main.ONOScli2.clusters() )
- clusters.append( main.ONOScli3.clusters() )
- clusters.append( main.ONOScli4.clusters() )
- clusters.append( main.ONOScli5.clusters() )
- clusters.append( main.ONOScli6.clusters() )
- clusters.append( main.ONOScli7.clusters() )
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].clusters,
+ name="clusters-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ clusters.append( t.result )
elapsed = time.time() - startTime
cliTime = time.time() - cliStart
@@ -1847,7 +1987,11 @@
onpass="Clusters view is consistent across all ONOS nodes",
onfail="ONOS nodes have different views of clusters" )
# there should always only be one cluster
- numClusters = len( json.loads( clusters[ 0 ] ) )
+ try:
+ numClusters = len( json.loads( clusters[ 0 ] ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing clusters[0]: " +
+ repr( clusters[0] ) )
clusterResults = main.FALSE
if numClusters == 1:
clusterResults = main.TRUE
@@ -1876,11 +2020,50 @@
if topoResult == main.TRUE:
main.log.report( "ONOS topology view matches Mininet topology" )
+ #FIXME: move this to an ONOS state case
+ main.step( "Checking ONOS nodes" )
+ nodesOutput = []
+ threads = []
+ for i in range( numControllers ):
+ t = main.Thread( target=CLIs[i].nodes,
+ name="nodes-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ nodesOutput.append( t.result )
+ ips = [ node.ip_address for node in nodes ]
+ for i in nodesOutput:
+ try:
+ current = json.loads( i )
+ for node in current:
+ if node['ip'] in ips: # node in nodes() output is in cell
+ if node['state'] == 'ACTIVE':
+ pass # as it should be
+ else:
+ main.log.error( "Error in ONOS node availability" )
+ main.log.error(
+ json.dumps( current,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ break
+ except ( ValueError, TypeError ):
+ main.log.error( "Error parsing nodes output" )
+ main.log.warn( repr( i ) )
+
def CASE9( self, main ):
"""
Link s3-s28 down
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# NOTE: You should probably run a topology check after this
linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
@@ -1896,7 +2079,7 @@
" seconds for link down to be discovered" )
time.sleep( linkSleep )
utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
- onpass="Link down succesful",
+ onpass="Link down successful",
onfail="Failed to bring link down" )
# TODO do some sort of check here
@@ -1905,6 +2088,11 @@
Link s3-s28 up
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# NOTE: You should probably run a topology check after this
linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
@@ -1920,7 +2108,7 @@
" seconds for link up to be discovered" )
time.sleep( linkSleep )
utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
- onpass="Link up succesful",
+ onpass="Link up successful",
onfail="Failed to bring link up" )
# TODO do some sort of check here
@@ -1930,6 +2118,11 @@
"""
# NOTE: You should probably run a topology check after this
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
@@ -1953,7 +2146,7 @@
if device and device[ 'available' ] is False:
result = main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Kill switch succesful",
+ onpass="Kill switch successful",
onfail="Failed to kill switch?" )
def CASE12( self, main ):
@@ -1962,6 +2155,18 @@
"""
# NOTE: You should probably run a topology check after this
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+ assert ONOS1Port, "ONOS1Port not defined"
+ assert ONOS2Port, "ONOS2Port not defined"
+ assert ONOS3Port, "ONOS3Port not defined"
+ assert ONOS4Port, "ONOS4Port not defined"
+ assert ONOS5Port, "ONOS5Port not defined"
+ assert ONOS6Port, "ONOS6Port not defined"
+ assert ONOS7Port, "ONOS7Port not defined"
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
switch = main.params[ 'kill' ][ 'switch' ]
@@ -1974,24 +2179,23 @@
main.step( "Add back " + switch )
main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
- # TODO: New dpid or same? Ask Thomas?
for peer in links:
main.Mininet1.addLink( switch, peer )
main.Mininet1.assignSwController( sw=switch.split( 's' )[ 1 ],
count=numControllers,
- ip1=ONOS1Ip,
+ ip1=nodes[ 0 ].ip_address,
port1=ONOS1Port,
- ip2=ONOS2Ip,
+ ip2=nodes[ 1 ].ip_address,
port2=ONOS2Port,
- ip3=ONOS3Ip,
+ ip3=nodes[ 2 ].ip_address,
port3=ONOS3Port,
- ip4=ONOS4Ip,
+ ip4=nodes[ 3 ].ip_address,
port4=ONOS4Port,
- ip5=ONOS5Ip,
+ ip5=nodes[ 4 ].ip_address,
port5=ONOS5Port,
- ip6=ONOS6Ip,
+ ip6=nodes[ 5 ].ip_address,
port6=ONOS6Port,
- ip7=ONOS7Ip,
+ ip7=nodes[ 6 ].ip_address,
port7=ONOS7Port )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch up to be discovered" )
@@ -2003,7 +2207,7 @@
if device and device[ 'available' ]:
result = main.TRUE
utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="add switch succesful",
+ onpass="add switch successful",
onfail="Failed to add switch?" )
def CASE13( self, main ):
@@ -2012,37 +2216,22 @@
"""
import os
import time
- # TODO: make use of this elsewhere
- ips = []
- ips.append( ONOS1Ip )
- ips.append( ONOS2Ip )
- ips.append( ONOS3Ip )
- ips.append( ONOS4Ip )
- ips.append( ONOS5Ip )
- ips.append( ONOS6Ip )
- ips.append( ONOS7Ip )
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
# printing colors to terminal
- colors = {}
- colors[ 'cyan' ] = '\033[96m'
- colors[ 'purple' ] = '\033[95m'
- colors[ 'blue' ] = '\033[94m'
- colors[ 'green' ] = '\033[92m'
- colors[ 'yellow' ] = '\033[93m'
- colors[ 'red' ] = '\033[91m'
- colors[ 'end' ] = '\033[0m'
+ colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
+ 'blue': '\033[94m', 'green': '\033[92m',
+ 'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
description = "Test Cleanup"
main.log.report( description )
main.case( description )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
- main.step( "Checking ONOS Logs for errors" )
- for i in range( 7 ):
- print colors[ 'purple' ] + "Checking logs for errors on " + \
- "ONOS" + str( i + 1 ) + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( ips[ i ] )
-
main.step( "Copying MN pcap and ONOS log files to test station" )
testname = main.TEST
teststationUser = main.params[ 'TESTONUSER' ]
@@ -2058,14 +2247,13 @@
# NOTE: must end in /
dstDir = "~/packet_captures/"
for f in logFiles:
- for i in range( 7 ):
- main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
- logFolder + f + " " +
+ for node in nodes:
+ main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
+ ":" + logFolder + f + " " +
teststationUser + "@" +
teststationIP + ":" +
dstDir + str( testname ) +
- "-ONOS" + str( i + 1 ) + "-" +
- f )
+ "-" + node.name + "-" + f )
main.ONOSbench.handle.expect( "\$" )
# std*.log's
@@ -2075,18 +2263,26 @@
# NOTE: must end in /
dstDir = "~/packet_captures/"
for f in logFiles:
- for i in range( 7 ):
- main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
- logFolder + f + " " +
+ for node in nodes:
+ main.ONOSbench.handle.sendline( "scp sdn@" + node.ip_address +
+ ":" + logFolder + f + " " +
teststationUser + "@" +
teststationIP + ":" +
dstDir + str( testname ) +
- "-ONOS" + str( i + 1 ) + "-" +
- f )
+ "-" + node.name + "-" + f )
main.ONOSbench.handle.expect( "\$" )
# sleep so scp can finish
time.sleep( 10 )
+
+ main.step( "Stopping Mininet" )
main.Mininet1.stopNet()
+
+ main.step( "Checking ONOS Logs for errors" )
+ for node in nodes:
+ print colors[ 'purple' ] + "Checking logs for errors on " + \
+ node.name + ":" + colors[ 'end' ]
+ print main.ONOSbench.checkLogs( node.ip_address )
+
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
@@ -2099,22 +2295,29 @@
"""
start election app on all onos nodes
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
leaderResult = main.TRUE
# install app on onos 1
main.log.info( "Install leadership election app" )
main.ONOScli1.featureInstall( "onos-app-election" )
+ leader = nodes[0].ip_address
# wait for election
# check for leader
- leader = main.ONOScli1.electionTestLeader()
+ leader1 = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
- if leader == ONOS1Ip:
+ if leader1 == leader:
# all is well
pass
- elif leader is None:
+ elif leader1 is None:
# No leader elected
main.log.report( "No leader was elected" )
leaderResult = main.FALSE
- elif leader == main.FALSE:
+ elif leader1 == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
@@ -2125,19 +2328,16 @@
# error in response
main.log.report(
"Unexpected response from electionTestLeader function:'" +
- str( leader ) +
- "'" )
+ str( leader1 ) + "'" )
leaderResult = main.FALSE
# install on other nodes and check for leader.
- # Should be onos1 and each app should show the same leader
- for controller in range( 2, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- node.featureInstall( "onos-app-election" )
- leaderN = node.electionTestLeader()
+ # Leader should be ONOS1 and each app should show the same leader
+ for cli in CLIs[ 1: ]:
+ cli.featureInstall( "onos-app-election" )
+ leaderN = cli.electionTestLeader()
# verify leader is ONOS1
- if leaderN == ONOS1Ip:
+ if leaderN == leader:
# all is well
pass
elif leaderN == main.FALSE:
@@ -2150,8 +2350,7 @@
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( "ONOS" + str( controller ) + " sees " +
- str( leaderN ) +
+ main.log.report( cli.name + " sees " + str( leaderN ) +
" as the leader of the election app. Leader" +
" should be " +
str( leader ) )
@@ -2169,6 +2368,12 @@
"""
Check that Leadership Election is still functional
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert CLIs, "CLIs not defined"
+ assert nodes, "nodes not defined"
+
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
main.log.report( description )
@@ -2177,28 +2382,18 @@
leader = main.ONOScli1.electionTestLeader()
# TODO: do some sanity checking on leader before using it
withdrawResult = main.FALSE
- if leader == ONOS1Ip:
- oldLeader = getattr( main, "ONOScli1" )
- elif leader == ONOS2Ip:
- oldLeader = getattr( main, "ONOScli2" )
- elif leader == ONOS3Ip:
- oldLeader = getattr( main, "ONOScli3" )
- elif leader == ONOS4Ip:
- oldLeader = getattr( main, "ONOScli4" )
- elif leader == ONOS5Ip:
- oldLeader = getattr( main, "ONOScli5" )
- elif leader == ONOS6Ip:
- oldLeader = getattr( main, "ONOScli6" )
- elif leader == ONOS7Ip:
- oldLeader = getattr( main, "ONOScli7" )
- elif leader is None or leader == main.FALSE:
+ if leader is None or leader == main.FALSE:
main.log.report(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
oldLeader = None
+ for i in range( len( CLIs ) ):
+ if leader == nodes[ i ].ip_address:
+ oldLeader = CLIs[ i ]
+ break
else:
- main.log.error( "Leader election --- why am I HERE?!?")
+ main.log.error( "Leader election, could not find current leader" )
if oldLeader:
withdrawResult = oldLeader.electionTestWithdraw()
utilities.assert_equals(
@@ -2209,21 +2404,17 @@
main.step( "Make sure new leader is elected" )
leaderList = []
- for controller in range( 1, numControllers + 1 ):
- # loop through ONOScli handlers
- node = getattr( main, ( 'ONOScli' + str( controller ) ) )
- leaderList.append( node.electionTestLeader() )
- for leaderN in leaderList:
+ for cli in CLIs:
+ leaderN = cli.electionTestLeader()
+ leaderList.append( leaderN )
if leaderN == leader:
- main.log.report(
- "ONOS" + str( controller ) +
- " still sees " + str( leader ) +
- " as leader after they withdrew" )
+ main.log.report( cli.name + " still sees " + str( leader ) +
+ " as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
- # means the app isn't loaded
+ # means the app isn't loaded
main.log.report( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
@@ -2240,6 +2431,7 @@
for n in range( len( leaderList ) ):
main.log.report( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
+ leaderResult = leaderResult and consistentLeader
if leaderResult:
main.log.report( "Leadership election tests passed( consistent " +
"view of leader across listeners and a new " +
diff --git a/TestON/tests/HATestSanity/HATestSanity.topo b/TestON/tests/HATestSanity/HATestSanity.topo
index 9305025..0855d4e 100644
--- a/TestON/tests/HATestSanity/HATestSanity.topo
+++ b/TestON/tests/HATestSanity/HATestSanity.topo
@@ -7,7 +7,8 @@
<password>onos_test</password>
<type>OnosDriver</type>
<connect_order>1</connect_order>
- <COMPONENTS> </COMPONENTS>
+ <COMPONENTS>
+ </COMPONENTS>
</ONOSbench>
<ONOScli1>
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index 9d9a3fa..d6e3daf 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -49,7 +49,7 @@
main.case( "Setting up test environment" )
# TODO: save all the timers and output them for plotting
- # load some vairables from the params file
+ # load some variables from the params file
PULLCODE = False
if main.params[ 'Git' ] == 'True':
PULLCODE = True
@@ -72,6 +72,7 @@
global ONOS7Ip
global ONOS7Port
global numControllers
+ numControllers = int( main.params[ 'num_controllers' ] )
ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
@@ -87,7 +88,6 @@
ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
- numControllers = int( main.params[ 'num_controllers' ] )
main.step( "Applying cell variable to environment" )
cellResult = main.ONOSbench.setCell( cellName )
@@ -167,6 +167,16 @@
Assign mastership to controllers
"""
import re
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert ONOS1Port, "ONOS1Port not defined"
+ assert ONOS2Port, "ONOS2Port not defined"
+ assert ONOS3Port, "ONOS3Port not defined"
+ assert ONOS4Port, "ONOS4Port not defined"
+ assert ONOS5Port, "ONOS5Port not defined"
+ assert ONOS6Port, "ONOS6Port not defined"
+ assert ONOS7Port, "ONOS7Port not defined"
main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
@@ -202,6 +212,9 @@
"""
import time
import json
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
# FIXME: we must reinstall intents until we have a persistant
# datastore!
main.log.report( "Adding host intents" )
@@ -216,15 +229,16 @@
# REACTIVE FWD test
pingResult = main.FALSE
- time1 = time.time()
- pingResult = main.Mininet1.pingall()
- utilities.assert_equals(
- expect=main.TRUE,
- actual=pingResult,
- onpass="Reactive Pingall test passed",
- onfail="Reactive Pingall failed, one or more ping pairs failed" )
- time2 = time.time()
- main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
+ for i in range(2): # Retry if pingall fails first time
+ time1 = time.time()
+ pingResult = main.Mininet1.pingall()
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=pingResult,
+ onpass="Reactive Pingall test passed",
+ onfail="Reactive Pingall failed, one or more ping pairs failed" )
+ time2 = time.time()
+ main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
# uninstall onos-app-fwd
main.log.info( "Uninstall reactive forwarding app" )
@@ -283,12 +297,15 @@
installedCheck = True
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
- for intent in json.loads( intents ): # Iter through intents of a node
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents" )
# add submitted intents not in the store
tmplist = [ i for i, s in intentStates ]
missingIntents = False
@@ -301,11 +318,62 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- # for node in nodes:
- # node.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
+
intentAddResult = bool( pingResult and hostResult and intentAddResult
and not missingIntents and installedCheck )
utilities.assert_equals(
@@ -327,13 +395,16 @@
intentStates = []
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
- for intent in json.loads( intents ):
- # Iter through intents of a node
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ # Iter through intents of a node
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents" )
# add submitted intents not in the store
tmplist = [ i for i, s in intentStates ]
for i in intentIds:
@@ -344,14 +415,70 @@
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
def CASE4( self, main ):
"""
Ping across added host intents
"""
import json
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
description = " Ping across added host intents"
main.log.report( description )
main.case( description )
@@ -370,11 +497,15 @@
main.log.report(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
- main.log.warn( "ONSO1 intents: " )
- main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
if PingResult == main.TRUE:
main.log.report(
"Intents have been installed correctly and verified by pings" )
@@ -392,21 +523,79 @@
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
# Iter through intents of a node
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
+
if not installedCheck:
- main.log.info( "Waiting 60 seconds to see if intent states change" )
+ main.log.info( "Waiting 60 seconds to see if the state of " +
+ "intents change" )
time.sleep( 60 )
# Print the intent states
intents = main.ONOScli1.intents()
@@ -414,26 +603,85 @@
main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
count = 0
# Iter through intents of a node
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
intentStates.sort()
for i, s in intentStates:
count += 1
main.log.info( "%-6s%-15s%-15s" %
( str( count ), str( i ), str( s ) ) )
- main.ONOScli1.leaders()
- main.ONOScli1.partitions()
- main.ONOScli1.pendingMap()
+ leaders = main.ONOScli1.leaders()
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ topics.append( "org.onosproject.election" )
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ else:
+ main.log.error( "leaders() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
def CASE5( self, main ):
"""
Reading state of ONOS
"""
import json
+ import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
@@ -441,7 +689,7 @@
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
- # We can then compare them with eachother and also with past states
+ # We can then compare them with each other and also with past states
main.step( "Get the Mastership of each switch from each controller" )
global mastershipState
@@ -628,6 +876,9 @@
The Failure case.
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
main.log.report( "Restart ONOS node" )
main.log.case( "Restart ONOS node" )
@@ -660,6 +911,9 @@
Check state after ONOS failure
"""
import json
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
main.case( "Running ONOS Constant State Tests" )
# Assert that each device has a master
@@ -850,6 +1104,9 @@
from sts.topology.teston_topology import TestONTopology
import json
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
description = "Compare ONOS Topology view to Mininet topology"
main.case( description )
@@ -1002,6 +1259,9 @@
Link s3-s28 down
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
# NOTE: You should probably run a topology check after this
linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
@@ -1026,6 +1286,9 @@
Link s3-s28 up
"""
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
# NOTE: You should probably run a topology check after this
linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
@@ -1051,6 +1314,9 @@
"""
# NOTE: You should probably run a topology check after this
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
@@ -1083,6 +1349,16 @@
"""
# NOTE: You should probably run a topology check after this
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
+ assert ONOS1Port, "ONOS1Port not defined"
+ assert ONOS2Port, "ONOS2Port not defined"
+ assert ONOS3Port, "ONOS3Port not defined"
+ assert ONOS4Port, "ONOS4Port not defined"
+ assert ONOS5Port, "ONOS5Port not defined"
+ assert ONOS6Port, "ONOS6Port not defined"
+ assert ONOS7Port, "ONOS7Port not defined"
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
switch = main.params[ 'kill' ][ 'switch' ]
@@ -1120,6 +1396,9 @@
"""
import os
import time
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
# printing colors to terminal
colors = {}
colors[ 'cyan' ] = '\033[96m'
@@ -1139,6 +1418,7 @@
print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
colors[ 'end' ]
print main.ONOSbench.checkLogs( ONOS1Ip )
+
main.step( "Copying MN pcap and ONOS log files to test station" )
testname = main.TEST
teststationUser = main.params[ 'TESTONUSER' ]
@@ -1189,6 +1469,9 @@
"""
start election app on all onos nodes
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
leaderResult = main.TRUE
# install app on onos 1
main.log.info( "Install leadership election app" )
@@ -1259,6 +1542,9 @@
"""
Check that Leadership Election is still functional
"""
+ assert numControllers, "numControllers not defined"
+ assert main, "main not defined"
+ assert utilities.assert_equals, "utilities.assert_equals not defined"
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
main.log.report( description )