Update HATests
- Clean up some core code( no functional changes)
- add retry to cli functions using the distributed primitives app
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index 7199280..a15bb8c 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -78,8 +78,6 @@
self.testResult = "Summary"
self.stepName = ""
self.stepCache = ""
- # make this into two lists? one for step names, one for results?
- # this way, the case result could be a true AND of these results
self.EXPERIMENTAL_MODE = False
self.test_target = None
self.lastcommand = None
diff --git a/TestON/core/testparser.py b/TestON/core/testparser.py
index 37f50f0..24b1ca2 100644
--- a/TestON/core/testparser.py
+++ b/TestON/core/testparser.py
@@ -16,7 +16,7 @@
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
- along with TestON. If not, see <http://www.gnu.org/licenses/>.
+ along with TestON. If not, see <http://www.gnu.org/licenses/>.
'''
@@ -26,21 +26,21 @@
def __init__(self,testFile):
try :
testFileHandler = open(testFile, 'r')
- except IOError:
+ except IOError:
print "No such file "+testFile
sys.exit(0)
-
+
testFileList = testFileHandler.readlines()
- self.testscript = testFileList
+ self.testscript = testFileList
self.caseCode = {}
self.caseBlock = ''
self.statementsList = []
- index = 0
+ index = 0
self.statementsList = []
#initialSpaces = len(line) -len(line.lstrip())
while index < len(testFileList):
testFileList[index] = re.sub("^\s{8}|^\s{4}", "", testFileList[index])
- # Skip multiline comments
+ # Skip multiline comments
if re.match('^(\'\'\')|^(\"\"\")',testFileList[index],0) :
index = index + 1
try :
@@ -48,16 +48,15 @@
index = index + 1
except IndexError,e:
print ''
-
- # skip empty lines and single line comments
+ # skip empty lines and single line comments
elif not re.match('#|^\s*$',testFileList[index],0):
self.statementsList.append(testFileList[index])
index = index + 1
-
+
def case_code(self):
- index = 0
- statementsList = self.statementsList
+ index = 0
+ statementsList = self.statementsList
while index < len(statementsList):
#print statementsList[index]
m= re.match('def\s+CASE(\d+)',statementsList[index],0)
@@ -76,18 +75,16 @@
except IndexError,e:
#print 'IndexError'
print ''
-
self.caseCode [str(m.group(1))] = self.caseBlock
#print "Case CODE "+self.caseCode [str(m.group(1))]
index = index + 1
-
- return self.caseCode
-
+ return self.caseCode
+
def step_code(self,caseStatements):
index = 0
- step = 0
- stepCode = {}
- step_flag = False
+ step = 0
+ stepCode = {}
+ step_flag = False
while index < len(caseStatements):
m= re.match('main\.step',caseStatements[index],0)
stepBlock = ''
@@ -99,13 +96,13 @@
while i < index :
block += caseStatements[i]
i = i + 1
- stepCode[step] = block
+ stepCode[step] = block
step = step + 1
- stepBlock= stepBlock + caseStatements[index]
+ stepBlock = stepBlock + caseStatements[index]
index = index + 1
try :
while not re.match('main\.step',caseStatements[index],0) :
- stepBlock= stepBlock + caseStatements[index]
+ stepBlock = stepBlock + caseStatements[index]
if index < len(caseStatements)-1:
index = index + 1
else :
@@ -121,11 +118,10 @@
if not step_flag :
stepCode[step] = "".join(caseStatements)
return stepCode
-
+
def getStepCode(self):
case_step_code = {}
case_block = self.case_code()
-
for case in case_block :
case_step_code[case] = {}
step_block = self.step_code(case_block[case])
diff --git a/TestON/drivers/common/cli/emulator/mininetclidriver.py b/TestON/drivers/common/cli/emulator/mininetclidriver.py
index ad50b9c..a58ffb9 100644
--- a/TestON/drivers/common/cli/emulator/mininetclidriver.py
+++ b/TestON/drivers/common/cli/emulator/mininetclidriver.py
@@ -1557,7 +1557,7 @@
if switchesJson == "": # if rest call fails
main.log.error(
self.name +
- ".compare_switches(): Empty JSON object given from ONOS" )
+ ".compareSwitches(): Empty JSON object given from ONOS" )
return main.FALSE
onos = switchesJson
onosDPIDs = []
@@ -1866,7 +1866,7 @@
if onosMAC == mnIntf[ 'hw_addr' ].lower() :
match = True
for ip in mnIntf[ 'ips' ]:
- if ip in onosHost[ 'ips' ]:
+ if ip in onosHost[ 'ipAddresses' ]:
pass # all is well
else:
# misssing ip
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index e95e5c1..0958531 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -92,7 +92,7 @@
return main.ERROR
else:
main.log.error( "Error, unexpected output in the ping file" )
- main.log.warn( outputs )
+ #main.log.warn( outputs )
return main.TRUE
def pingLong( self, **pingParams ):
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index a67205c..c7e25e0 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -1820,7 +1820,7 @@
if flow.get( 'state' ) != 'ADDED' and flow.get( 'state' ) != \
'PENDING_ADD':
main.log.info( self.name + ": flow Id: " +
- flow.get( 'flowId' ) +
+ flow.get( 'id' ) +
" | state:" + flow.get( 'state' ) )
returnValue = main.FALSE
return returnValue
@@ -3129,6 +3129,20 @@
try:
cmdStr = "set-test-add " + str( setName ) + " " + str( values )
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
positiveMatch = "\[(.*)\] was added to the set " + str( setName )
negativeMatch = "\[(.*)\] was already in set " + str( setName )
@@ -3184,6 +3198,20 @@
else:
cmdStr += str( setName ) + " " + str( values )
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
if clear:
@@ -3269,6 +3297,20 @@
cmdStr = "set-test-get "
cmdStr += setName + " " + values
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
@@ -3334,7 +3376,7 @@
Required arguments:
setName - The name of the set to remove from.
returns:
- The integer value of the size returned or
+ The integer value of the size returned or
None on error
"""
try:
@@ -3348,6 +3390,20 @@
cmdStr = "set-test-get -s "
cmdStr += setName
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
match = re.search( pattern, output )
@@ -3450,6 +3506,20 @@
cmdStr += "-i "
cmdStr += counter
output = self.sendline( cmdStr )
+ try:
+ # TODO: Maybe make this less hardcoded
+ # ConsistentMap Exceptions
+ assert "org.onosproject.store.service" not in output
+ # Node not leader
+ assert "java.lang.IllegalStateException" not in output
+ except AssertionError:
+ main.log.error( "Error in processing 'set-test-add' " +
+ "command: " + str( output ) )
+ retryTime = 30 # Conservative time, given by Madan
+ main.log.info( "Waiting " + str( retryTime ) +
+ "seconds before retrying." )
+ time.sleep( retryTime ) # Due to change in mastership
+ output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
pattern = counter + " was incremented to (\d+)"
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 1f153ee..583c2cc 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -926,7 +926,7 @@
"""
try:
self.handle.sendline( "" )
- self.handle.expect( "\$" )
+ self.handle.expect( "\$", timeout=60 )
self.handle.sendline( "onos-uninstall " + str( nodeIp ) )
self.handle.expect( "\$" )
@@ -935,6 +935,9 @@
# onos-uninstall command does not return any text
return main.TRUE
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": Timeout in onosUninstall" )
+ return main.FALSE
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index 2499625..bb69fcc 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -49,6 +49,9 @@
main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -115,26 +118,27 @@
onpass="Mininet Started",
onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
# values of 1 or 3 are good
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
-
- main.step( "Using mvn clean and install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
- utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if gitPullResult == main.TRUE:
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
# GRAPHS
# NOTE: important params here:
# job = name of Jenkins job
@@ -227,10 +231,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onosIsupResult and cliResults )
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -253,8 +255,12 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device. Then manually assign" +\
+ " mastership to specific ONOS nodes using" +\
+ " 'device-role'"
main.step( "Assign switches to controllers" )
# TODO: rewrite this function to take lists of ips and ports?
@@ -388,8 +394,6 @@
"""
Assign intents
"""
- # FIXME: we must reinstall intents until we have a persistant
- # datastore!
import time
import json
assert numControllers, "numControllers not defined"
@@ -397,11 +401,14 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Adding host intents" )
+ # NOTE: we must reinstall intents until we have a persistant intent
+ # datastore!
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
main.step( "Install reactive forwarding app" )
@@ -410,6 +417,7 @@
onpass="Install fwd successful",
onfail="Install fwd failed" )
+ main.step( "Check app ids" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -429,6 +437,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -451,8 +461,10 @@
utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
onpass="Uninstall fwd successful",
onfail="Uninstall fwd failed" )
- main.step( "Check app ids check" )
+
+ main.step( "Check app ids" )
threads = []
+ appCheck2 = main.TRUE
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].appToIDCheck,
name="appToIDCheck-" + str( i ),
@@ -462,15 +474,15 @@
for t in threads:
t.join()
- appCheck = appCheck and t.result
- if appCheck != main.TRUE:
+ appCheck2 = appCheck2 and t.result
+ if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -513,7 +525,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -608,13 +623,11 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
correct = True
@@ -747,9 +760,11 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -783,83 +798,102 @@
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +\
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -942,6 +976,37 @@
main.log.error( repr( pendingMap ) )
main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.report(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -956,7 +1021,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -1378,7 +1442,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if not host.get( 'ips', [ ] ):
+ if not host.get( 'ipAddresses', [ ] ):
main.log.error( "DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
@@ -1489,6 +1553,7 @@
"""
The Failure case.
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -1505,6 +1570,7 @@
main.case( "Restart entire ONOS cluster" )
main.step( "Killing ONOS nodes" )
killResults = main.TRUE
+ killTime = time.time()
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
killResults = killResults and killed
@@ -1542,6 +1608,29 @@
utilities.assert_equals( expect=main.TRUE, actual=cliResults,
onpass="ONOS cli started",
onfail="ONOS clis did not restart" )
+
+ # Grab the time of restart so we chan check how long the gossip
+ # protocol has had time to work
+ main.restartTime = time.time() - killTime
+ main.log.debug( "Restart time: " + str( main.restartTime ) )
+ '''
+ # FIXME: revisit test plan for election with madan
+ # Rerun for election on restarted nodes
+ run1 = CLIs[0].electionTestRun()
+ run2 = CLIs[1].electionTestRun()
+ run3 = CLIs[2].electionTestRun()
+ ...
+ ...
+ runResults = run1 and run2 and run3
+ utilities.assert_equals( expect=main.TRUE, actual=runResults,
+ onpass="Reran for election",
+ onfail="Failed to rerun for election" )
+ '''
+ # TODO: Make this configurable
+ time.sleep( 60 )
+ main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+ main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+ main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
def CASE7( self, main ):
"""
@@ -1630,6 +1719,7 @@
elif rolesResults and not consistentMastership:
mastershipCheck = main.TRUE
+ '''
description2 = "Compare switch roles from before failure"
main.step( description2 )
try:
@@ -1655,15 +1745,13 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed" )
+ '''
# NOTE: we expect mastership to change on controller failure
- mastershipCheck = mastershipCheck and consistentMastership
main.step( "Get the intents and compare across all nodes" )
ONOSIntents = []
@@ -1769,7 +1857,7 @@
sameIntents = main.TRUE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
else:
@@ -1804,8 +1892,6 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
@@ -1849,7 +1935,7 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
- # error in response
+ # error in response
main.log.report( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
@@ -1864,10 +1950,6 @@
main.log.error(
"Inconsistent view of leader for the election test app" )
# TODO: print the list
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1891,16 +1973,26 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- for node in nodes:
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
+ main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
portsResults = main.TRUE
linksResults = main.TRUE
@@ -1949,9 +2041,9 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ips on controller" +
+ "DEBUG:Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -2142,8 +2234,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
@@ -2420,6 +2510,15 @@
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ timerLog.write( "Restart\n" )
+ timerLog.write( str( main.restartTime ) )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
+
def CASE14( self, main ):
"""
start election app on all onos nodes
@@ -2430,10 +2529,17 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- leaderResult = main.TRUE
main.case("Start Leadership Election app")
main.step( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
leaders = []
for cli in CLIs:
cli.electionTestRun()
@@ -2445,19 +2551,23 @@
str( leader ) + "'" )
leaderResult = main.FALSE
leaders.append( leader )
- if len( set( leaders ) ) != 1:
- leaderResult = main.FALSE
- main.log.error( "Results of electionTestLeader is order of CLIs:" +
- str( leaders ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
def CASE15( self, main ):
"""
@@ -2474,12 +2584,44 @@
description = "Check that Leadership Election is still functional"
main.log.report( description )
main.case( description )
+ # NOTE: Need to re-run since being a canidate is not persistant
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
+ leaders = []
+ for cli in CLIs:
+ cli.electionTestRun()
+ for cli in CLIs:
+ leader = cli.electionTestLeader()
+ if leader is None or leader == main.FALSE:
+ main.log.report( cli.name + ": Leader for the election app " +
+ "should be an ONOS node, instead got '" +
+ str( leader ) + "'" )
+ leaderResult = main.FALSE
+ leaders.append( leader )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=leaderResult,
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
+
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
# do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2495,8 +2637,8 @@
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
# FIXME: use threads
@@ -2535,11 +2677,6 @@
main.log.report( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was elected when the old leader " +
- "resigned )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -2558,6 +2695,7 @@
onpass="App re-ran for election",
onfail="App failed to run for election" )
+ main.step( "Leader did not change when old leader re-ran" )
afterRun = main.ONOScli1.electionTestLeader()
# verify leader didn't just change
if afterRun == leaderList[ 0 ]:
@@ -2572,14 +2710,6 @@
onfail="Something went wrong with Leadership election after " +
"the old leader re-ran for election" )
- case15Result = withdrawResult and leaderResult and runResult and\
- afterResult
- utilities.assert_equals(
- expect=main.TRUE,
- actual=case15Result,
- onpass="Leadership election is still functional",
- onfail="Leadership Election is no longer functional" )
-
def CASE16( self, main ):
"""
Install Distributed Primitives app
@@ -2613,6 +2743,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
+ time.sleep (5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2656,11 +2787,13 @@
main.step( "Increment and get a default counter on each node" )
pCounters = []
threads = []
+ addedPValues = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
name="counterIncrement-" + str( i ),
args=[ pCounterName ] )
pCounterValue += 1
+ addedPValues.append( pCounterValue )
threads.append( t )
t.start()
@@ -2669,8 +2802,12 @@
pCounters.append( t.result )
# Check that counter incremented numController times
pCounterResults = True
- for i in range( numControllers ):
- pCounterResults and ( i + 1 ) in pCounters
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=pCounterResults,
onpass="Default counter incremented",
@@ -2679,6 +2816,7 @@
main.step( "Increment and get an in memory counter on each node" )
iCounters = []
+ addedIValues = []
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
@@ -2686,6 +2824,7 @@
args=[ iCounterName ],
kwargs={ "inMemory": True } )
iCounterValue += 1
+ addedIValues.append( iCounterValue )
threads.append( t )
t.start()
@@ -2694,8 +2833,12 @@
iCounters.append( t.result )
# Check that counter incremented numController times
iCounterResults = True
- for i in range( numControllers ):
- iCounterResults and ( i + 1 ) in iCounters
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=iCounterResults,
onpass="In memory counter incremented",
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 0dd40ed..256561a 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -49,6 +49,9 @@
main.log.report( "ONOS HA test: Restart minority of ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -115,26 +118,27 @@
onpass="Mininet Started",
onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
# values of 1 or 3 are good
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
-
- main.step( "Using mvn clean and install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
- utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if gitPullResult == main.TRUE:
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
# GRAPHS
# NOTE: important params here:
# job = name of Jenkins job
@@ -227,10 +231,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onosIsupResult and cliResults )
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -253,8 +255,12 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device. Then manually assign" +\
+ " mastership to specific ONOS nodes using" +\
+ " 'device-role'"
main.step( "Assign switches to controllers" )
# TODO: rewrite this function to take lists of ips and ports?
@@ -286,8 +292,6 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -395,11 +399,12 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Adding host intents" )
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
main.step( "Install reactive forwarding app" )
@@ -408,6 +413,7 @@
onpass="Install fwd successful",
onfail="Install fwd failed" )
+ main.step( "Check app ids" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -427,6 +433,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -449,8 +457,10 @@
utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
onpass="Uninstall fwd successful",
onfail="Uninstall fwd failed" )
- main.step( "Check app ids check" )
+
+ main.step( "Check app ids" )
threads = []
+ appCheck2 = main.TRUE
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].appToIDCheck,
name="appToIDCheck-" + str( i ),
@@ -460,15 +470,15 @@
for t in threads:
t.join()
- appCheck = appCheck and t.result
- if appCheck != main.TRUE:
+ appCheck2 = appCheck2 and t.result
+ if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -511,7 +521,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -606,13 +619,11 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
correct = True
@@ -745,9 +756,11 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -781,83 +794,102 @@
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +\
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -940,6 +972,37 @@
main.log.error( repr( pendingMap ) )
main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.report(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -954,7 +1017,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -1376,7 +1438,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if not host.get( 'ips', [ ] ):
+ if not host.get( 'ipAddresses', [ ] ):
main.log.error( "DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
@@ -1495,6 +1557,7 @@
assert nodes, "nodes not defined"
main.case( "Restart minority of ONOS nodes" )
main.step( "Killing 3 ONOS nodes" )
+ killTime = time.time()
# TODO: Randomize these nodes or base this on partitions
# TODO: use threads in this case
killResults = main.ONOSbench.onosKill( nodes[0].ip_address )
@@ -1533,7 +1596,28 @@
# Grab the time of restart so we chan check how long the gossip
# protocol has had time to work
- main.restartTime = time.time()
+ main.restartTime = time.time() - killTime
+ main.log.debug( "Restart time: " + str( main.restartTime ) )
+ '''
+ # FIXME: revisit test plan for election with madan
+ # Rerun for election on restarted nodes
+ run1 = CLIs[0].electionTestRun()
+ run2 = CLIs[1].electionTestRun()
+ run3 = CLIs[2].electionTestRun()
+ runResults = run1 and run2 and run3
+ utilities.assert_equals( expect=main.TRUE, actual=runResults,
+ onpass="Reran for election",
+ onfail="Failed to rerun for election" )
+ '''
+ # TODO: MAke this configurable. Also, we are breaking the above timer
+ time.sleep( 60 )
+ main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+ main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+ main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
+ time.sleep( 200 )
+ main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
+ main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
+ main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
def CASE7( self, main ):
"""
@@ -1569,7 +1653,6 @@
main.step( "Read device roles from ONOS" )
ONOSMastership = []
- mastershipCheck = main.FALSE
consistentMastership = True
rolesResults = True
threads = []
@@ -1619,9 +1702,9 @@
sort_keys=True,
indent=4,
separators=( ',', ': ' ) ) )
- elif rolesResults and not consistentMastership:
- mastershipCheck = main.TRUE
+ # NOTE: we expect mastership to change on controller failure
+ '''
description2 = "Compare switch roles from before failure"
main.step( description2 )
try:
@@ -1647,15 +1730,12 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Mastership of Switches was not changed",
onfail="Mastership of some switches changed" )
- # NOTE: we expect mastership to change on controller failure
- mastershipCheck = consistentMastership
+ '''
main.step( "Get the intents and compare across all nodes" )
ONOSIntents = []
@@ -1760,7 +1840,7 @@
sameIntents = main.TRUE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
else:
@@ -1795,8 +1875,6 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
@@ -1841,7 +1919,7 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == main.FALSE:
- # error in response
+ # error in response
main.log.report( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
@@ -1861,10 +1939,6 @@
main.log.error(
"Inconsistent view of leader for the election test app" )
# TODO: print the list
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1888,16 +1962,26 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- for node in nodes:
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
+ main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
portsResults = main.TRUE
linksResults = main.TRUE
@@ -1946,7 +2030,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -2139,8 +2223,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
@@ -2417,6 +2499,15 @@
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ timerLog.write( "Restart\n" )
+ timerLog.write( str( main.restartTime ) )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
+
def CASE14( self, main ):
"""
start election app on all onos nodes
@@ -2427,10 +2518,17 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- leaderResult = main.TRUE
main.case("Start Leadership Election app")
main.step( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
leaders = []
for cli in CLIs:
cli.electionTestRun()
@@ -2442,19 +2540,23 @@
str( leader ) + "'" )
leaderResult = main.FALSE
leaders.append( leader )
- if len( set( leaders ) ) != 1:
- leaderResult = main.FALSE
- main.log.error( "Results of electionTestLeader is order of CLIs:" +
- str( leaders ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
def CASE15( self, main ):
"""
@@ -2471,12 +2573,29 @@
description = "Check that Leadership Election is still functional"
main.log.report( description )
main.case( description )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ leaders = []
+ for cli in CLIs:
+ leader = cli.electionTestLeader()
+ leaders.append( leader )
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
+
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
# do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2492,8 +2611,8 @@
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
# FIXME: use threads
@@ -2532,11 +2651,6 @@
main.log.report( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was elected when the old leader " +
- "resigned )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -2555,6 +2669,7 @@
onpass="App re-ran for election",
onfail="App failed to run for election" )
+ main.step( "Leader did not change when old leader re-ran" )
afterRun = main.ONOScli1.electionTestLeader()
# verify leader didn't just change
if afterRun == leaderList[ 0 ]:
@@ -2569,14 +2684,6 @@
onfail="Something went wrong with Leadership election after " +
"the old leader re-ran for election" )
- case15Result = withdrawResult and leaderResult and runResult and\
- afterResult
- utilities.assert_equals(
- expect=main.TRUE,
- actual=case15Result,
- onpass="Leadership election is still functional",
- onfail="Leadership Election is no longer functional" )
-
def CASE16( self, main ):
"""
Install Distributed Primitives app
@@ -2610,6 +2717,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
+ time.sleep (5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2653,11 +2761,13 @@
main.step( "Increment and get a default counter on each node" )
pCounters = []
threads = []
+ addedPValues = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
name="counterIncrement-" + str( i ),
args=[ pCounterName ] )
pCounterValue += 1
+ addedPValues.append( pCounterValue )
threads.append( t )
t.start()
@@ -2666,8 +2776,12 @@
pCounters.append( t.result )
# Check that counter incremented numController times
pCounterResults = True
- for i in range( numControllers ):
- pCounterResults and ( i + 1 ) in pCounters
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=pCounterResults,
onpass="Default counter incremented",
@@ -2676,6 +2790,7 @@
main.step( "Increment and get an in memory counter on each node" )
iCounters = []
+ addedIValues = []
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
@@ -2683,6 +2798,7 @@
args=[ iCounterName ],
kwargs={ "inMemory": True } )
iCounterValue += 1
+ addedIValues.append( iCounterValue )
threads.append( t )
t.start()
@@ -2691,8 +2807,12 @@
iCounters.append( t.result )
# Check that counter incremented numController times
iCounterResults = True
- for i in range( numControllers ):
- iCounterResults and ( i + 1 ) in iCounters
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=iCounterResults,
onpass="In memory counter incremented",
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
index 3c4541c..e4658e7 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
@@ -120,12 +120,27 @@
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
- main.step( "Using mvn clean & install" )
+ main.step( "Using mvn clean and install" )
cleanInstallResult = main.ONOSbench.cleanInstall()
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+ # GRAPHS
+ # NOTE: important params here:
+ # job = name of Jenkins job
+ # Plot Name = Plot-HA, only can be used if multiple plots
+ # index = The number of the graph under plot name
+ job = "HANetworkPartition"
+ graphs = '<ac:structured-macro ac:name="html">\n'
+ graphs += '<ac:plain-text-body><![CDATA[\n'
+ graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
+ '/plot/getPlot?index=0&width=500&height=300"' +\
+ 'noborder="0" width="500" height="300" scrolling="yes" '+\
+ 'seamless="seamless"></iframe>\n'
+ graphs += ']]></ac:plain-text-body>\n'
+ graphs += '</ac:structured-macro>\n'
+ main.log.wiki(graphs)
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage()
@@ -218,6 +233,7 @@
onfail="Test startup NOT successful" )
if case1Result == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -1414,7 +1430,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host[ 'ips' ] == []:
+ if host[ 'ipAddresses' ] == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index dace37a..2440ec6 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -17,7 +17,6 @@
#CASE15: Check that Leadership Election is still functional
#CASE16: Install Distributed Primitives app
#CASE17: Check for basic functionality with distributed primitives
- #1,2,8,3,4,5,14,[6],8,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13
#1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13
<testcases>1,2,8,3,4,5,14,16,17,[6],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index 8bc21b0..fe513ac 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -49,6 +49,9 @@
"""
main.log.report( "ONOS HA Sanity test - initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -115,26 +118,27 @@
onpass="Mininet Started",
onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
# values of 1 or 3 are good
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
-
- main.step( "Using mvn clean and install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
- utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if gitPullResult == main.TRUE:
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
# GRAPHS
# NOTE: important params here:
# job = name of Jenkins job
@@ -229,10 +233,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onosIsupResult and cliResults )
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -255,8 +257,12 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device. Then manually assign" +\
+ " mastership to specific ONOS nodes using" +\
+ " 'device-role'"
main.step( "Assign switches to controllers" )
# TODO: rewrite this function to take lists of ips and ports?
@@ -397,11 +403,12 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- main.log.report( "Adding host intents" )
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
main.step( "Install reactive forwarding app" )
@@ -410,6 +417,7 @@
onpass="Install fwd successful",
onfail="Install fwd failed" )
+ main.step( "Check app ids" )
appCheck = main.TRUE
threads = []
for i in range( numControllers ):
@@ -429,6 +437,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -451,8 +461,15 @@
utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
onpass="Uninstall fwd successful",
onfail="Uninstall fwd failed" )
- main.step( "Check app ids check" )
+ '''
+ main.Mininet1.handle.sendline( "py [ h.cmd( \"arping -c 1 10.1.1.1 \" ) for h in net.hosts ] ")
+ import time
+ time.sleep(60)
+ '''
+
+ main.step( "Check app ids" )
threads = []
+ appCheck2 = main.TRUE
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].appToIDCheck,
name="appToIDCheck-" + str( i ),
@@ -462,15 +479,15 @@
for t in threads:
t.join()
- appCheck = appCheck and t.result
- if appCheck != main.TRUE:
+ appCheck2 = appCheck2 and t.result
+ if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck2,
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -513,7 +530,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -608,13 +628,11 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
correct = True
@@ -747,9 +765,11 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -783,83 +803,102 @@
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +\
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -942,6 +981,46 @@
main.log.error( repr( pendingMap ) )
main.log.debug( CLIs[0].flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.report(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
+
+
+
+ '''
+ #DEBUG
+ if PingResult == main.FALSE:
+ import time
+ time.sleep( 100000 )
+ '''
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -956,7 +1035,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -1378,7 +1456,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if not host.get( 'ips', [ ] ):
+ if not host.get( 'ipAddresses', [ ] ):
main.log.error( "DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
@@ -1615,8 +1693,6 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -1727,7 +1803,7 @@
sameIntents = main.TRUE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
else:
@@ -1762,8 +1838,6 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
@@ -1821,10 +1895,6 @@
main.log.report( cli.name + " sees " + str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1848,16 +1918,26 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- for node in nodes:
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ for node in nodes:
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
+ main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
portsResults = main.TRUE
linksResults = main.TRUE
@@ -1906,7 +1986,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -2099,8 +2179,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
# FIXME: move this to an ONOS state case
main.step( "Checking ONOS nodes" )
@@ -2378,11 +2456,11 @@
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
try:
- gossipIntentLog = open( main.logdir + "/Timers.csv", 'w')
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
# Overwrite with empty line and close
- gossipIntentLog.write( "Gossip Intents\n" )
- gossipIntentLog.write( str( gossipTime ) )
- gossipIntentLog.close()
+ timerLog.write( "Gossip Intents\n" )
+ timerLog.write( str( gossipTime ) )
+ timerLog.close()
except NameError, e:
main.log.exception(e)
@@ -2396,10 +2474,17 @@
assert CLIs, "CLIs not defined"
assert nodes, "nodes not defined"
- leaderResult = main.TRUE
main.case("Start Leadership Election app")
main.step( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.TRUE
leaders = []
for cli in CLIs:
cli.electionTestRun()
@@ -2411,19 +2496,23 @@
str( leader ) + "'" )
leaderResult = main.FALSE
leaders.append( leader )
- if len( set( leaders ) ) != 1:
- leaderResult = main.FALSE
- main.log.error( "Results of electionTestLeader is order of CLIs:" +
- str( leaders ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
def CASE15( self, main ):
"""
@@ -2440,12 +2529,29 @@
description = "Check that Leadership Election is still functional"
main.log.report( description )
main.case( description )
+
+ main.step( "Check that each node shows the same leader" )
+ sameLeader = main.TRUE
+ leaders = []
+ for cli in CLIs:
+ leader = cli.electionTestLeader()
+ leaders.append( leader )
+ if len( set( leaders ) ) != 1:
+ sameLeader = main.FALSE
+ main.log.error( "Results of electionTestLeader is order of CLIs:" +
+ str( leaders ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=sameLeader,
+ onpass="Leadership is consistent for the election topic",
+ onfail="Nodes have different leaders" )
+
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
# do some sanity checking on leader before using it
withdrawResult = main.FALSE
if leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2461,8 +2567,8 @@
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
# FIXME: use threads
@@ -2501,11 +2607,6 @@
main.log.report( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was elected when the old leader " +
- "resigned )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -2524,6 +2625,7 @@
onpass="App re-ran for election",
onfail="App failed to run for election" )
+ main.step( "Leader did not change when old leader re-ran" )
afterRun = main.ONOScli1.electionTestLeader()
# verify leader didn't just change
if afterRun == leaderList[ 0 ]:
@@ -2538,14 +2640,6 @@
onfail="Something went wrong with Leadership election after " +
"the old leader re-ran for election" )
- case15Result = withdrawResult and leaderResult and runResult and\
- afterResult
- utilities.assert_equals(
- expect=main.TRUE,
- actual=case15Result,
- onpass="Leadership election is still functional",
- onfail="Leadership Election is no longer functional" )
-
def CASE16( self, main ):
"""
Install Distributed Primitives app
@@ -2579,6 +2673,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
+ time.sleep (5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2622,11 +2717,13 @@
main.step( "Increment and get a default counter on each node" )
pCounters = []
threads = []
+ addedPValues = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
name="counterIncrement-" + str( i ),
args=[ pCounterName ] )
pCounterValue += 1
+ addedPValues.append( pCounterValue )
threads.append( t )
t.start()
@@ -2635,8 +2732,12 @@
pCounters.append( t.result )
# Check that counter incremented numController times
pCounterResults = True
- for i in range( numControllers ):
- pCounterResults and ( i + 1 ) in pCounters
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=pCounterResults,
onpass="Default counter incremented",
@@ -2645,6 +2746,7 @@
main.step( "Increment and get an in memory counter on each node" )
iCounters = []
+ addedIValues = []
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
@@ -2652,6 +2754,7 @@
args=[ iCounterName ],
kwargs={ "inMemory": True } )
iCounterValue += 1
+ addedIValues.append( iCounterValue )
threads.append( t )
t.start()
@@ -2660,8 +2763,12 @@
iCounters.append( t.result )
# Check that counter incremented numController times
iCounterResults = True
- for i in range( numControllers ):
- iCounterResults and ( i + 1 ) in iCounters
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=iCounterResults,
onpass="In memory counter incremented",
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
index d027c36..f057592 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
@@ -16,7 +16,7 @@
#CASE15: Check that Leadership Election is still functional
#1,2,8,3,4,5,14,[6],8,3,7,4,15,9,8,4,10,8,4,11,8,4,12,8,4,13
#extra hosts test 1,2,8,11,8,12,8
- <testcases>1,2,8,3,4,5,14,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+ <testcases>1,2,8,3,4,5,14,15,16,17,[6],8,3,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
<ENV>
<cellName>HA</cellName>
</ENV>
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index 0f68a02..7194a3b 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -49,6 +49,9 @@
main.log.report( "ONOS Single node cluster restart " +
"HA test - initialization" )
main.case( "Setting up test environment" )
+ main.caseExplaination = "Setup the test environment including " +\
+ "installing ONOS, starting Mininet and ONOS" +\
+ "cli sessions."
# TODO: save all the timers and output them for plotting
# load some variables from the params file
@@ -92,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -115,26 +118,27 @@
onpass="Mininet Started",
onfail="Error starting Mininet" )
- main.step( "Compiling the latest version of ONOS" )
+ main.step( "Git checkout and pull " + gitBranch )
if PULLCODE:
- main.step( "Git checkout and pull " + gitBranch )
main.ONOSbench.gitCheckout( gitBranch )
gitPullResult = main.ONOSbench.gitPull()
# values of 1 or 3 are good
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
-
- main.step( "Using mvn clean and install" )
- cleanInstallResult = main.ONOSbench.cleanInstall()
- utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
else:
main.log.warn( "Did not pull new code so skipping mvn " +
"clean install" )
main.ONOSbench.getVersion( report=True )
+
+ main.step( "Using mvn clean install" )
+ cleanInstallResult = main.TRUE
+ if gitPullResult == main.TRUE:
+ cleanInstallResult = main.ONOSbench.cleanInstall()
+ utilities.assert_equals( expect=main.TRUE,
+ actual=cleanInstallResult,
+ onpass="MCI successful",
+ onfail="MCI failed" )
# GRAPHS
# NOTE: important params here:
# job = name of Jenkins job
@@ -171,8 +175,6 @@
onos1Isup = main.ONOSbench.isup( ONOS1Ip )
if onos1Isup:
break
- if not onos1Isup:
- main.log.report( "ONOS1 didn't start!" )
utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
onpass="ONOS startup successful",
onfail="ONOS startup failed" )
@@ -191,18 +193,7 @@
port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
- appCheck = main.TRUE
- threads = []
- for i in range( numControllers ):
- t = main.Thread( target=CLIs[i].appToIDCheck,
- name="appToIDCheck-" + str( i ),
- args=[] )
- threads.append( t )
- t.start()
-
- for t in threads:
- t.join()
- appCheck = appCheck and t.result
+ appCheck = main.ONOScli1.appToIDCheck()
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
main.log.warn( CLIs[0].appIDs() )
@@ -210,11 +201,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- case1Result = ( cleanInstallResult and packageResult and
- cellResult and verifyResult and onosInstallResult
- and onos1Isup and cliResults )
-
- if case1Result == main.FALSE:
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
main.cleanup()
main.exit()
@@ -234,8 +222,10 @@
assert ONOS6Port, "ONOS6Port not defined"
assert ONOS7Port, "ONOS7Port not defined"
- main.log.report( "Assigning switches to controllers" )
main.case( "Assigning Controllers" )
+ main.caseExplaination = "Assign switches to ONOS using 'ovs-vsctl' " +\
+ "and check that an ONOS node becomes the " +\
+ "master of the device."
main.step( "Assign switches to controllers" )
for i in range( 1, 29 ):
@@ -255,7 +245,7 @@
else:
mastershipCheck = main.FALSE
if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
+ main.log.info( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -340,13 +330,14 @@
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- # FIXME: we must reinstall intents until we have a persistant
- # datastore!
- main.log.report( "Adding host intents" )
+ # NOTE: we must reinstall intents until we have a persistant intent
+ # datastore!
main.case( "Adding host Intents" )
-
- main.step( "Discovering Hosts( Via pingall for now )" )
- # FIXME: Once we have a host discovery mechanism, use that instead
+ main.caseExplaination = "Discover hosts by using pingall then " +\
+ "assign predetermined host-to-host intents." +\
+ " After installation, check that the intent" +\
+ " is distributed to all nodes and the state" +\
+ " is INSTALLED"
# install onos-app-fwd
main.step( "Install reactive forwarding app" )
@@ -355,6 +346,7 @@
onpass="Install fwd successful",
onfail="Install fwd failed" )
+ main.step( "Check app ids" )
appCheck = main.ONOScli1.appToIDCheck()
if appCheck != main.TRUE:
main.log.warn( CLIs[0].apps() )
@@ -363,6 +355,8 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
+ main.step( "Discovering Hosts( Via pingall for now )" )
+ # FIXME: Once we have a host discovery mechanism, use that instead
# REACTIVE FWD test
pingResult = main.FALSE
for i in range(2): # Retry if pingall fails first time
@@ -385,6 +379,8 @@
utilities.assert_equals( expect=main.TRUE, actual=uninstallResult,
onpass="Uninstall fwd successful",
onfail="Uninstall fwd failed" )
+
+ main.step( "Check app ids" )
appCheck2 = main.ONOScli1.appToIDCheck()
if appCheck2 != main.TRUE:
main.log.warn( CLIs[0].apps() )
@@ -393,7 +389,7 @@
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- main.step( "Add host intents" )
+ main.step( "Add host intents via cli" )
intentIds = []
# TODO: move the host numbers to params
# Maybe look at all the paths we ping?
@@ -435,7 +431,10 @@
except ( ValueError, TypeError ):
main.log.warn( repr( hosts ) )
hostResult = main.FALSE
- # FIXME: DEBUG
+ utilities.assert_equals( expect=main.TRUE, actual=hostResult,
+ onpass="Found a host id for each host",
+ onfail="Error looking up host ids" )
+
intentStart = time.time()
onosIds = main.ONOScli1.getAllIntentsId()
main.log.info( "Submitted intents: " + str( intentIds ) )
@@ -530,13 +529,11 @@
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- intentAddResult = bool( pingResult and hostResult and intentAddResult
- and not missingIntents and installedCheck )
- utilities.assert_equals(
- expect=True,
- actual=intentAddResult,
- onpass="Pushed host intents to ONOS",
- onfail="Error in pushing host intents to ONOS" )
+ intentAddResult = bool( intentAddResult and not missingIntents and
+ installedCheck )
+ if not intentAddResult:
+ main.log.error( "Error in pushing host intents to ONOS" )
+
main.step( "Intent Anti-Entropy dispersion" )
for i in range(100):
correct = True
@@ -667,9 +664,11 @@
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- description = " Ping across added host intents"
- main.log.report( description )
- main.case( description )
+ main.case( "Verify connectivity by sendind traffic across Intents" )
+ main.caseExplaination = "Ping across added host intents to check " +\
+ "functionality and check the state of " +\
+ "the intent"
+ main.step( "Ping across added host intents" )
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -695,7 +694,7 @@
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
if PingResult == main.TRUE:
- main.log.report(
+ main.log.info(
"Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
@@ -703,83 +702,102 @@
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
+ main.step( "Check Intent state" )
installedCheck = True
- if PingResult is not main.TRUE:
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
- count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
- leaders = main.ONOScli1.leaders()
- try:
- if leaders:
- parsedLeaders = json.loads( leaders )
- main.log.warn( json.dumps( parsedLeaders,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # check for all intent partitions
- # check for election
- topics = []
- for i in range( 14 ):
- topics.append( "intent-partition-" + str( i ) )
- # FIXME: this should only be after we start the app
- topics.append( "org.onosproject.election" )
- main.log.debug( topics )
- ONOStopics = [ j['topic'] for j in parsedLeaders ]
- for topic in topics:
- if topic not in ONOStopics:
- main.log.error( "Error: " + topic +
- " not in leaders" )
- else:
- main.log.error( "leaders() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing leaders" )
- main.log.error( repr( leaders ) )
- partitions = main.ONOScli1.partitions()
- try:
- if partitions :
- parsedPartitions = json.loads( partitions )
- main.log.warn( json.dumps( parsedPartitions,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check for a leader in all paritions
- # TODO check for consistency among nodes
- else:
- main.log.error( "partitions() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing partitions" )
- main.log.error( repr( partitions ) )
- pendingMap = main.ONOScli1.pendingMap()
- try:
- if pendingMap :
- parsedPending = json.loads( pendingMap )
- main.log.warn( json.dumps( parsedPending,
- sort_keys=True,
- indent=4,
- separators=( ',', ': ' ) ) )
- # TODO check something here?
- else:
- main.log.error( "pendingMap() returned None" )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing pending map" )
- main.log.error( repr( pendingMap ) )
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ utilities.assert_equals( expect=True, actual=installedCheck,
+ onpass="Intents are all INSTALLED",
+ onfail="Intents are not all in " +\
+ "INSTALLED state" )
+
+ main.step( "Check leadership of topics" )
+ leaders = main.ONOScli1.leaders()
+ topicCheck = main.TRUE
+ try:
+ if leaders:
+ parsedLeaders = json.loads( leaders )
+ main.log.warn( json.dumps( parsedLeaders,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # check for all intent partitions
+ # check for election
+ # TODO: Look at Devices as topics now that it uses this system
+ topics = []
+ for i in range( 14 ):
+ topics.append( "intent-partition-" + str( i ) )
+ # FIXME: this should only be after we start the app
+ # FIXME: topics.append( "org.onosproject.election" )
+ # Print leaders output
+ main.log.debug( topics )
+ ONOStopics = [ j['topic'] for j in parsedLeaders ]
+ for topic in topics:
+ if topic not in ONOStopics:
+ main.log.error( "Error: " + topic +
+ " not in leaders" )
+ topicCheck = main.FALSE
+ else:
+ main.log.error( "leaders() returned None" )
+ topicCheck = main.FALSE
+ except ( ValueError, TypeError ):
+ topicCheck = main.FALSE
+ main.log.exception( "Error parsing leaders" )
+ main.log.error( repr( leaders ) )
+ # TODO: Check for a leader of these topics
+ utilities.assert_equals( expect=main.TRUE, actual=topicCheck,
+ onpass="intent Partitions is in leaders",
+ onfail="Some topics were lost " )
+ # Print partitions
+ partitions = main.ONOScli1.partitions()
+ try:
+ if partitions :
+ parsedPartitions = json.loads( partitions )
+ main.log.warn( json.dumps( parsedPartitions,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check for a leader in all paritions
+ # TODO check for consistency among nodes
+ else:
+ main.log.error( "partitions() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing partitions" )
+ main.log.error( repr( partitions ) )
+ # Print Pending Map
+ pendingMap = main.ONOScli1.pendingMap()
+ try:
+ if pendingMap :
+ parsedPending = json.loads( pendingMap )
+ main.log.warn( json.dumps( parsedPending,
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ # TODO check something here?
+ else:
+ main.log.error( "pendingMap() returned None" )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing pending map" )
+ main.log.error( repr( pendingMap ) )
if not installedCheck:
main.log.info( "Waiting 60 seconds to see if the state of " +
@@ -862,6 +880,37 @@
main.log.error( repr( pendingMap ) )
main.log.debug( main.ONOScli1.flows( jsonFormat=False ) )
+ main.step( "Wait a minute then ping again" )
+ PingResult = main.TRUE
+ for i in range( 8, 18 ):
+ ping = main.Mininet1.pingHost( src="h" + str( i ),
+ target="h" + str( i + 10 ) )
+ PingResult = PingResult and ping
+ if ping == main.FALSE:
+ main.log.warn( "Ping failed between h" + str( i ) +
+ " and h" + str( i + 10 ) )
+ elif ping == main.TRUE:
+ main.log.info( "Ping test passed!" )
+ # Don't set PingResult or you'd override failures
+ if PingResult == main.FALSE:
+ main.log.report(
+ "Intents have not been installed correctly, pings failed." )
+ # TODO: pretty print
+ main.log.warn( "ONOS1 intents: " )
+ try:
+ tmpIntents = main.ONOScli1.intents()
+ main.log.warn( json.dumps( json.loads( tmpIntents ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.warn( repr( tmpIntents ) )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=PingResult,
+ onpass="Intents have been installed correctly and pings work",
+ onfail="Intents have not been installed correctly, pings failed." )
+
def CASE5( self, main ):
"""
Reading state of ONOS
@@ -873,7 +922,6 @@
# assumes that sts is already in you PYTHONPATH
from sts.topology.teston_topology import TestONTopology
- main.log.report( "Setting up and gathering data for current state" )
main.case( "Setting up and gathering data for current state" )
# The general idea for this test case is to pull the state of
# ( intents,flows, topology,... ) from each ONOS node
@@ -961,7 +1009,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -1074,7 +1122,9 @@
iCounterValue = 0
main.log.report( "Restart ONOS node" )
- main.log.case( "Restart ONOS node" )
+ main.case( "Restart ONOS node" )
+ main.caseExplaination = "Killing ONOS process and restart cli " +\
+ "sessions once onos is up."
main.step( "Killing ONOS processes" )
killResult = main.ONOSbench.onosKill( ONOS1Ip )
start = time.time()
@@ -1105,6 +1155,8 @@
main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
str( elapsed ) )
time.sleep( 5 )
+ # rerun on election apps
+ main.ONOScli1.electionTestRun()
def CASE7( self, main ):
"""
@@ -1115,7 +1167,6 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
main.case( "Running ONOS Constant State Tests" )
-
main.step( "Check that each switch has a master" )
# Assert that each device has a master
rolesNotNull = main.ONOScli1.rolesNotNull()
@@ -1162,8 +1213,6 @@
else:
main.log.warn( "Mastership of switch %s changed" % switchDPID )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Mastership of Switches was not changed" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -1207,9 +1256,9 @@
sameIntents = main.TRUE
if intentState and intentState == ONOS1Intents:
sameIntents = main.TRUE
- main.log.report( "Intents are consistent with before failure" )
+ main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
- # what the aceptable states are
+ # what the acceptable states are
else:
try:
main.log.warn( "ONOS1 intents: " )
@@ -1241,14 +1290,13 @@
if FlowTables == main.FALSE:
main.log.info( "Differences in flow table for switch: s" +
str( i + 1 ) )
- if FlowTables == main.TRUE:
- main.log.report( "No changes were found in the flow tables" )
utilities.assert_equals(
expect=main.TRUE,
actual=FlowTables,
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables" )
+ main.step( "Leadership Election is still functional" )
# Test of LeadershipElection
leader = ONOS1Ip
@@ -1264,7 +1312,7 @@
# all is well
pass
elif leaderN == main.FALSE:
- # error in response
+ # error in response
main.log.report( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
@@ -1275,25 +1323,12 @@
str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a new " +
- "leader was re-elected if applicable )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
onpass="Leadership election passed",
onfail="Something went wrong with Leadership election" )
- result = ( mastershipCheck and intentCheck and FlowTables and
- rolesNotNull and leaderResult )
- result = int( result )
- if result == main.TRUE:
- main.log.report( "Constant State Tests Passed" )
- utilities.assert_equals( expect=main.TRUE, actual=result,
- onpass="Constant State Tests Passed",
- onfail="Constant state tests failed" )
-
def CASE8( self, main ):
"""
Compare topo
@@ -1309,15 +1344,24 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- description = "Compare ONOS Topology view to Mininet topology"
- main.case( description )
- main.log.report( description )
+ main.case( "Compare ONOS Topology view to Mininet topology" )
+ main.caseExplaination = "Compare topology objects between Mininet" +\
+ " and ONOS"
main.step( "Create TestONTopology object" )
- ctrls = []
- node = main.ONOS1
- temp = ( node, node.name, node.ip_address, 6633 )
- ctrls.append( temp )
- MNTopo = TestONTopology( main.Mininet1, ctrls )
+ try:
+ ctrls = []
+ node = main.ONOS1
+ temp = ( node, node.name, node.ip_address, 6633 )
+ ctrls.append( temp )
+ MNTopo = TestONTopology( main.Mininet1, ctrls )
+ except Exception:
+ objResult = main.FALSE
+ else:
+ objResult = main.TRUE
+ utilities.assert_equals( expect=main.TRUE, actual=objResult,
+ onpass="Created TestONTopology object",
+ onfail="Exception while creating " +
+ "TestONTopology object" )
main.step( "Comparing ONOS topology to MN" )
devicesResults = main.TRUE
@@ -1344,7 +1388,7 @@
for controller in range( 0, len( hosts ) ):
controllerStr = str( controller + 1 )
for host in hosts[ controller ]:
- if host is None or host.get( 'ips', [] ) == []:
+ if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
"DEBUG:Error with host ips on controller" +
controllerStr + ": " + str( host ) )
@@ -1445,8 +1489,6 @@
utilities.assert_equals( expect=main.TRUE, actual=topoResult,
onpass="Topology Check Test successful",
onfail="Topology Check Test NOT successful" )
- if topoResult == main.TRUE:
- main.log.report( "ONOS topology view matches Mininet topology" )
def CASE9( self, main ):
"""
@@ -1662,10 +1704,17 @@
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
- leaderResult = main.TRUE
main.case("Start Leadership Election app")
main.step( "Install leadership election app" )
- main.ONOScli1.activateApp( "org.onosproject.election" )
+ appResult = main.ONOScli1.activateApp( "org.onosproject.election" )
+ utilities.assert_equals(
+ expect=main.TRUE,
+ actual=appResult,
+ onpass="Election app installed",
+ onfail="Something went wrong with installing Leadership election" )
+
+ main.step( "Run for election on each node" )
+ leaderResult = main.ONOScli1.electionTestRun()
# check for leader
leader = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
@@ -1674,32 +1723,27 @@
pass
elif leader is None:
# No leader elected
- main.log.report( "No leader was elected" )
+ main.log.error( "No leader was elected" )
leaderResult = main.FALSE
elif leader == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with electionTestLeader" +
+ main.log.error( "Something is wrong with electionTestLeader" +
" function, check the error logs" )
leaderResult = main.FALSE
else:
# error in response
- main.log.report(
+ main.log.error(
"Unexpected response from electionTestLeader function:'" +
str( leader ) +
"'" )
leaderResult = main.FALSE
-
- if leaderResult:
- main.log.report( "Leadership election tests passed( consistent " +
- "view of leader across listeners and a leader " +
- "was elected )" )
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
- onpass="Leadership election passed",
- onfail="Something went wrong with Leadership election" )
+ onpass="Successfully ran for leadership",
+ onfail="Failed to run for leadership" )
def CASE15( self, main ):
"""
@@ -1726,13 +1770,15 @@
oldLeader = None
else:
main.log.error( "Leader election --- why am I HERE?!?")
+ leaderResult = main.FALSE
+ oldLeader = None
if oldLeader:
withdrawResult = oldLeader.electionTestWithdraw()
utilities.assert_equals(
expect=main.TRUE,
actual=withdrawResult,
- onpass="App was withdrawn from election",
- onfail="App was not withdrawn from election" )
+ onpass="Node was withdrawn from election",
+ onfail="Node was not withdrawn from election" )
main.step( "Make sure new leader is elected" )
leaderN = main.ONOScli1.electionTestLeader()
@@ -1750,9 +1796,7 @@
elif leaderN is None:
main.log.info(
"There is no leader after the app withdrew from election" )
- if leaderResult:
- main.log.report( "Leadership election tests passed( There is no " +
- "leader after the old leader resigned )" )
+ leaderResult = main.TRUE
utilities.assert_equals(
expect=main.TRUE,
actual=leaderResult,
@@ -1770,20 +1814,21 @@
actual=runResult,
onpass="App re-ran for election",
onfail="App failed to run for election" )
- leader = main.ONOScli1.electionTestLeader()
+
+ main.step( "Node became leader when it ran for election" )
+ afterRun = main.ONOScli1.electionTestLeader()
# verify leader is ONOS1
- if leader == ONOS1Ip:
- leaderResult = main.TRUE
+ if afterRun == ONOS1Ip:
+ afterResult = main.TRUE
else:
- leaderResult = main.FALSE
- # TODO: assert on run and withdraw results?
+ afterResult = main.FALSE
utilities.assert_equals(
expect=main.TRUE,
- actual=leaderResult,
- onpass="Leadership election passed",
- onfail="ONOS1's election app was not leader after it re-ran " +
- "for election" )
+ actual=afterResult,
+ onpass="Old leader successfully re-ran for election",
+ onfail="Something went wrong with Leadership election after " +
+ "the old leader re-ran for election" )
def CASE16( self, main ):
"""
@@ -1861,11 +1906,13 @@
main.step( "Increment and get a default counter on each node" )
pCounters = []
threads = []
+ addedPValues = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
name="counterIncrement-" + str( i ),
args=[ pCounterName ] )
pCounterValue += 1
+ addedPValues.append( pCounterValue )
threads.append( t )
t.start()
@@ -1874,8 +1921,12 @@
pCounters.append( t.result )
# Check that counter incremented numController times
pCounterResults = True
- for i in range( numControllers ):
- pCounterResults and ( i + 1 ) in pCounters
+ for i in addedPValues:
+ tmpResult = i in pCounters
+ pCounterResults = pCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in partitioned "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=pCounterResults,
onpass="Default counter incremented",
@@ -1884,6 +1935,7 @@
main.step( "Increment and get an in memory counter on each node" )
iCounters = []
+ addedIValues = []
threads = []
for i in range( numControllers ):
t = main.Thread( target=CLIs[i].counterTestIncrement,
@@ -1891,6 +1943,7 @@
args=[ iCounterName ],
kwargs={ "inMemory": True } )
iCounterValue += 1
+ addedIValues.append( iCounterValue )
threads.append( t )
t.start()
@@ -1899,8 +1952,12 @@
iCounters.append( t.result )
# Check that counter incremented numController times
iCounterResults = True
- for i in range( numControllers ):
- iCounterResults and ( i + 1 ) in iCounters
+ for i in addedIValues:
+ tmpResult = i in iCounters
+ iCounterResults = iCounterResults and tmpResult
+ if not tmpResult:
+ main.log.error( str( i ) + " is not in the in-memory "
+ "counter incremented results" )
utilities.assert_equals( expect=True,
actual=iCounterResults,
onpass="In memory counter incremented",