Merge branch 'master' of https://github.com/opennetworkinglab/ONLabTest into FuncPlatform
diff --git a/TestON/core/teston.py b/TestON/core/teston.py
index 8187e4e..fc85c55 100644
--- a/TestON/core/teston.py
+++ b/TestON/core/teston.py
@@ -288,13 +288,12 @@
self.stepCache += "No Result\n"
#self.stepCache += "No Result <ac:emoticon ac:name=\"warning\" /></li>\n"
self.stepResults.append(self.STEPRESULT)
- except TypeError,e:
- print "\nException in the following section of code: " +\
- str(testCaseNumber) + "." + str(step) + ": " +\
- self.stepName
+ except StandardError as e:
+ self.log.exception( "\nException in the following section of" +
+ " code: " + str(testCaseNumber) + "." +
+ str(step) + ": " + self.stepName )
#print code[testCaseNumber][step]
self.stepCount = self.stepCount + 1
- self.log.exception(e)
self.logger.updateCaseResults(self)
#WIKI results
self.log.wiki( "<ul>" )
diff --git a/TestON/drivers/common/cli/emulator/remotemininetdriver.py b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
index 0004cfc..3b0b01b 100644
--- a/TestON/drivers/common/cli/emulator/remotemininetdriver.py
+++ b/TestON/drivers/common/cli/emulator/remotemininetdriver.py
@@ -91,8 +91,9 @@
elif re.search( "found multiple mininet", outputs ):
return main.ERROR
else:
+ # TODO: Parse for failed pings, give some truncated output
main.log.error( "Error, unexpected output in the ping file" )
- #main.log.warn( outputs )
+ main.log.warn( outputs )
return main.TRUE
def arping( self, host="", ip="10.128.20.211" ):
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index ec660eb..5207f1e 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -281,7 +281,11 @@
lvlStr = "--level=" + level
self.handle.sendline( "" )
- self.handle.expect( "onos>" )
+ i = self.handle.expect( [ "onos>", pexpect.TIMEOUT ] )
+ # TODO: look for bash prompt as well
+ if i == 1:
+ self.handle.sendline( "" )
+ self.handle.expect( "onos>" )
self.handle.sendline( "log:log " + lvlStr + " " + cmdStr )
self.handle.expect( "log:log" )
self.handle.expect( "onos>" )
@@ -290,7 +294,10 @@
if re.search( "Error", response ):
return main.FALSE
return main.TRUE
-
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.cleanup()
+ main.exit()
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
@@ -3158,7 +3165,6 @@
else:
main.log.error( self.name + ": setTestAdd did not" +
" match expected output" )
- main.log.debug( self.name + " expected: " + pattern )
main.log.debug( self.name + " actual: " + repr( output ) )
return main.ERROR
except AssertionError:
@@ -3446,42 +3452,31 @@
main.cleanup()
main.exit()
- def counters( self ):
+ def counters( self, jsonFormat=True ):
"""
Command to list the various counters in the system.
returns:
- A dict containing the counters in the system with the counter
- names being the keys and the value of the counters being the
- values OR
+ if jsonFormat, a string of the json object returned by the cli
+ command
+ if not jsonFormat, the normal string output of the cli command
None on error
"""
- #FIXME: JSON FORMAT
try:
counters = {}
cmdStr = "counters"
+ if jsonFormat:
+ cmdStr += " -j"
output = self.sendline( cmdStr )
assert "Error executing command" not in output
main.log.info( self.name + ": " + output )
- for line in output.splitlines():
- match = re.search( "name=(\S+) value=(\d+)", line )
- if match:
- key = match.group( 1 )
- value = int( match.group( 2 ) )
- counters[key] = value
- else:
- main.log.error( self.name + ": counters did not match " +
- "expected output" )
- main.log.debug( self.name + " expected: " + pattern )
- main.log.debug( self.name + " actual: " + repr( output ) )
- return None
- return counters
+ return output
except AssertionError:
main.log.error( "Error in processing 'counters' command: " +
str( output ) )
- return main.ERROR
+ return None
except TypeError:
main.log.exception( self.name + ": Object not as expected" )
- return main.ERROR
+ return None
except pexpect.EOF:
main.log.error( self.name + ": EOF exception found" )
main.log.error( self.name + ": " + self.handle.before )
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 7593e87..832e139 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -485,8 +485,10 @@
main.exit()
def getBranchName( self ):
+ main.log.info( "self.home = " )
+ main.log.info( self.home )
self.handle.sendline( "cd " + self.home )
- self.handle.expect( "ONOS\$" )
+ self.handle.expect( self.home + "\$" )
self.handle.sendline( "git name-rev --name-only HEAD" )
self.handle.expect( "git name-rev --name-only HEAD" )
self.handle.expect( "\$" )
@@ -1519,13 +1521,18 @@
except Exception:
main.log.exception( "Copying files failed" )
- def checkLogs( self, onosIp ):
+ def checkLogs( self, onosIp, restart=False):
"""
runs onos-check-logs on the given onos node
+ If restart is True, use the old version of onos-check-logs which
+ does not print the full stacktrace, but shows the entire log file,
+ including across restarts
returns the response
"""
try:
cmd = "onos-check-logs " + str( onosIp )
+ if restart:
+ cmd += " old"
self.handle.sendline( cmd )
self.handle.expect( cmd )
self.handle.expect( "\$" )
diff --git a/TestON/drivers/common/cli/quaggaclidriver.py b/TestON/drivers/common/cli/quaggaclidriver.py
index f391e3a..7e3df2b 100644
--- a/TestON/drivers/common/cli/quaggaclidriver.py
+++ b/TestON/drivers/common/cli/quaggaclidriver.py
@@ -219,8 +219,8 @@
intentsJsonObj = json.loads( getIntentsResult )
for intent in intentsJsonObj:
- if intent[ 'appId' ] != "org.onosproject.sdnip":
- continue
+ #if intent[ 'appId' ] != "org.onosproject.sdnip":
+ # continue
if intent[ 'type' ] == "MultiPointToSinglePointIntent" \
and intent[ 'state' ] == 'INSTALLED':
egress = str( intent[ 'egress' ][ 'device' ] ) + ":" \
@@ -242,6 +242,26 @@
intents.append( intent )
return sorted( intents )
+ # This method calculates the MultiPointToSinglePointIntent number installed
+ def extractActualRouteIntentNum( self, getIntentsResult ):
+ intentsJsonObj = json.loads( getIntentsResult )
+ num = 0
+ for intent in intentsJsonObj:
+ if intent[ 'type' ] == "MultiPointToSinglePointIntent" \
+ and intent[ 'state' ] == 'INSTALLED':
+ num = num + 1
+ return num
+
+ # This method calculates the PointToPointIntent number installed
+ def extractActualBgpIntentNum( self, getIntentsResult ):
+ intentsJsonObj = json.loads( getIntentsResult )
+ num = 0
+ for intent in intentsJsonObj:
+ if intent[ 'type' ] == "PointToPointIntent" \
+ and intent[ 'state' ] == 'INSTALLED':
+ num = num + 1
+ return num
+
# This method extracts all actual BGP intents from ONOS CLI
def extractActualBgpIntents( self, getIntentsResult ):
intents = []
@@ -250,8 +270,8 @@
intentsJsonObj = json.loads( getIntentsResult )
for intent in intentsJsonObj:
- if intent[ 'appId' ] != "org.onosproject.sdnip":
- continue
+ #if intent[ 'appId' ] != "org.onosproject.sdnip":
+ # continue
if intent[ 'type' ] == "PointToPointIntent" \
and "protocol=6" in str( intent[ 'selector' ] ):
ingress = str( intent[ 'ingress' ][ 'device' ] ) + ":" \
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
index 500c3ad..390facf 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.params
@@ -3,9 +3,10 @@
<ENV>
<cellName>HA</cellName>
</ENV>
- <Git>False</Git>
+ <Git> False </Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
index bb69fcc..50bf06f 100644
--- a/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
+++ b/TestON/tests/HATestClusterRestart/HATestClusterRestart.py
@@ -46,7 +46,7 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS HA test: Restart all ONOS nodes - " +
+ main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
main.caseExplaination = "Setup the test environment including " +\
@@ -95,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -126,15 +126,15 @@
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean install" )
cleanInstallResult = main.TRUE
- if gitPullResult == main.TRUE:
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
utilities.assert_equals( expect=main.TRUE,
actual=cleanInstallResult,
onpass="MCI successful",
@@ -145,10 +145,12 @@
# Plot Name = Plot-HA, only can be used if multiple plots
# index = The number of the graph under plot name
job = "HAClusterRestart"
+ plotName = "Plot-HA"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
- '/plot/getPlot?index=0&width=500&height=300"' +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
'noborder="0" width="500" height="300" scrolling="yes" ' +\
'seamless="seamless"></iframe>\n'
graphs += ']]></ac:plain-text-body>\n'
@@ -177,7 +179,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
@@ -204,12 +206,13 @@
onpass="ONOS cli startup successful",
onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
appCheck = main.TRUE
@@ -292,8 +295,6 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
@@ -538,7 +539,6 @@
pass # intent submitted is in onos
else:
intentAddResult = False
- # FIXME: DEBUG
if intentAddResult:
intentStop = time.time()
else:
@@ -639,7 +639,15 @@
main.log.debug( "Intents in " + cli.name + ": " +
str( sorted( onosIds ) ) )
if sorted( ids ) != sorted( intentIds ):
+ # IF intents are missing
correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ correct = False
+ break
if correct:
break
else:
@@ -777,7 +785,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -789,9 +797,6 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
@@ -799,31 +804,37 @@
onfail="Intents have not been installed correctly, pings failed." )
main.step( "Check Intent state" )
- installedCheck = True
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ installedCheck = False
count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- # Print states
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
+ while not installedCheck and count < 40:
+ installedCheck = True
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
utilities.assert_equals( expect=True, actual=installedCheck,
onpass="Intents are all INSTALLED",
- onfail="Intents are not all in " +\
+ onfail="Intents are not all in " +
"INSTALLED state" )
main.step( "Check leadership of topics" )
@@ -894,11 +905,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
-
+ main.log.info( "Waiting 60 seconds to see if the state of " +
+ "intents change" )
+ time.sleep( 60 )
if not installedCheck:
- main.log.info( "Waiting 60 seconds to see if the state of " +
- "intents change" )
- time.sleep( 60 )
# Print the intent states
intents = main.ONOScli1.intents()
intentStates = []
@@ -974,9 +984,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- main.log.debug( CLIs[0].flows( jsonFormat=False ) )
-
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
+ # the wait is above
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -989,7 +1000,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -1068,7 +1079,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1082,7 +1093,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1130,7 +1141,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1143,11 +1154,11 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1231,7 +1242,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1255,7 +1266,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1383,6 +1394,14 @@
for t in threads:
t.join()
ports.append( t.result )
+ try:
+ # FIXME: DEBUG
+ main.log.debug( json.dumps( json.loads( t.result ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except:
+ main.log.debug( repr( t.result ) )
links = []
threads = []
for i in range( numControllers ):
@@ -1418,14 +1437,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1461,12 +1480,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1568,6 +1587,15 @@
iCounterValue = 0
main.case( "Restart entire ONOS cluster" )
+ # FIXME: DEBUG
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ # FIXME: DEBUG
+
main.step( "Killing ONOS nodes" )
killResults = main.TRUE
killTime = time.time()
@@ -1584,7 +1612,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
onosIsupResult = onosIsupResult and started
if onosIsupResult == main.TRUE:
break
@@ -1608,25 +1636,25 @@
utilities.assert_equals( expect=main.TRUE, actual=cliResults,
onpass="ONOS cli started",
onfail="ONOS clis did not restart" )
-
+
# Grab the time of restart so we chan check how long the gossip
# protocol has had time to work
main.restartTime = time.time() - killTime
main.log.debug( "Restart time: " + str( main.restartTime ) )
- '''
+
# FIXME: revisit test plan for election with madan
# Rerun for election on restarted nodes
- run1 = CLIs[0].electionTestRun()
- run2 = CLIs[1].electionTestRun()
- run3 = CLIs[2].electionTestRun()
- ...
- ...
- runResults = run1 and run2 and run3
+ runResults = main.TRUE
+ for cli in CLIs:
+ run = CLIs[0].electionTestRun()
+ if run != main.TRUE:
+ main.log.error( "Error running for election on " + cli.name )
+ runResults = runResults and run
utilities.assert_equals( expect=main.TRUE, actual=runResults,
onpass="Reran for election",
onfail="Failed to rerun for election" )
- '''
- # TODO: Make this configurable
+
+ # TODO: Make this configurable
time.sleep( 60 )
main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
@@ -1683,7 +1711,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1697,7 +1725,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1773,7 +1801,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1786,7 +1814,7 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
@@ -1854,22 +1882,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1898,8 +1945,8 @@
onpass="No changes were found in the flow tables",
onfail="Changes were found in the flow tables" )
- main.step( "Check the continuous pings to ensure that no packets " +
- "were dropped during component failure" )
+ # main.step( "Check the continuous pings to ensure that no packets " +
+ # "were dropped during component failure" )
main.Mininet2.pingKill( main.params[ 'TESTONUSER' ],
main.params[ 'TESTONIP' ] )
LossInPings = main.FALSE
@@ -1917,12 +1964,13 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
- utilities.assert_equals(
- expect=main.FALSE,
- actual=LossInPings,
- onpass="No Loss of connectivity",
- onfail="Loss of dataplane connectivity detected" )
+ main.log.info( "No loss of dataplane connectivity" )
+ # utilities.assert_equals(
+ # expect=main.FALSE,
+ # actual=LossInPings,
+ # onpass="No Loss of connectivity",
+ # onfail="Loss of dataplane connectivity detected" )
+
# NOTE: Since intents are not persisted with IntnentStore,
# we expect loss in dataplane connectivity
LossInPings = main.FALSE
@@ -1936,14 +1984,13 @@
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.report( cli.name +
- " shows no leader for the election-app was" +
- " elected after the old one died" )
+ main.log.error( cli.name +
+ " shows no leader for the election-app." )
leaderResult = main.FALSE
if len( set( leaderList ) ) != 1:
leaderResult = main.FALSE
@@ -2003,11 +2050,12 @@
main.step( "Collecting topology information from ONOS" )
startTime = time.time()
# Give time for Gossip to work
- while topoResult == main.FALSE and elapsed < 60:
+ while topoResult == main.FALSE and elapsed < 120:
count += 1
if count > 1:
# TODO: Deprecate STS usage
MNTopo = TestONTopology( main.Mininet1, ctrls )
+ time.sleep( 1 )
cliStart = time.time()
devices = []
threads = []
@@ -2058,6 +2106,14 @@
for t in threads:
t.join()
ports.append( t.result )
+ try:
+ # FIXME: DEBUG
+ main.log.debug( json.dumps( json.loads( t.result ),
+ sort_keys=True,
+ indent=4,
+ separators=( ',', ': ' ) ) )
+ except:
+ main.log.debug( repr( t.result ) )
links = []
threads = []
for i in range( numControllers ):
@@ -2085,6 +2141,7 @@
elapsed = time.time() - startTime
cliTime = time.time() - cliStart
+ print "Elapsed time: " + str( elapsed )
print "CLI time: " + str( cliTime )
for controller in range( numControllers ):
@@ -2157,13 +2214,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2184,13 +2241,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2291,7 +2348,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2320,7 +2376,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2348,14 +2403,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2394,11 +2448,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2447,9 +2499,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2505,7 +2555,7 @@
for node in nodes:
print colors[ 'purple' ] + "Checking logs for errors on " + \
node.name + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( node.ip_address )
+ print main.ONOSbench.checkLogs( node.ip_address, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
@@ -2513,8 +2563,9 @@
try:
timerLog = open( main.logdir + "/Timers.csv", 'w')
# Overwrite with empty line and close
- timerLog.write( "Restart\n" )
- timerLog.write( str( main.restartTime ) )
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
timerLog.close()
except NameError, e:
main.log.exception(e)
@@ -2546,7 +2597,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2582,7 +2633,6 @@
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
# NOTE: Need to re-run since being a canidate is not persistant
main.step( "Run for election on each node" )
@@ -2593,7 +2643,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2647,14 +2697,14 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
@@ -2671,10 +2721,10 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
utilities.assert_equals(
@@ -2714,6 +2764,7 @@
"""
Install Distributed Primitives app
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -2743,7 +2794,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
- time.sleep (5 ) # To allow all nodes to activate
+ time.sleep( 5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2803,7 +2854,7 @@
# Check that counter incremented numController times
pCounterResults = True
for i in addedPValues:
- tmpResult = i in pCounters
+ tmpResult = i in pCounters
pCounterResults = pCounterResults and tmpResult
if not tmpResult:
main.log.error( str( i ) + " is not in partitioned "
@@ -2873,28 +2924,41 @@
main.step( "Counters we added have the correct values" )
correctResults = main.TRUE
for i in range( numControllers ):
- current = onosCounters[i]
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
try:
- pValue = current.get( pCounterName )
- iValue = current.get( iCounterName )
- if pValue == pCounterValue:
- main.log.info( "Partitioned counter value is correct" )
- else:
- main.log.error( "Partitioned counter value is incorrect," +
- " expected value: " + str( pCounterValue )
- + " current value: " + str( pValue ) )
- correctResults = main.FALSE
- if iValue == iCounterValue:
- main.log.info( "In memory counter value is correct" )
- else:
- main.log.error( "In memory counter value is incorrect, " +
- "expected value: " + str( iCounterValue ) +
- " current value: " + str( iValue ) )
- correctResults = main.FALSE
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
except AttributeError, e:
main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
"is not as expected" )
correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=correctResults,
onpass="Added counters are correct",
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
index ca1f915..956d889 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.params
@@ -6,6 +6,7 @@
<Git>False</Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
index 256561a..af96bb5 100644
--- a/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
+++ b/TestON/tests/HATestMinorityRestart/HATestMinorityRestart.py
@@ -46,7 +46,7 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS HA test: Restart minority of ONOS nodes - " +
+ main.log.info( "ONOS HA test: Restart minority of ONOS nodes - " +
"initialization" )
main.case( "Setting up test environment" )
main.caseExplaination = "Setup the test environment including " +\
@@ -95,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -126,15 +126,15 @@
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean install" )
cleanInstallResult = main.TRUE
- if gitPullResult == main.TRUE:
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
utilities.assert_equals( expect=main.TRUE,
actual=cleanInstallResult,
onpass="MCI successful",
@@ -145,10 +145,12 @@
# Plot Name = Plot-HA, only can be used if multiple plots
# index = The number of the graph under plot name
job = "HAMinorityRestart"
+ plotName = "Plot-HA"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
- '/plot/getPlot?index=0&width=500&height=300"' +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
'noborder="0" width="500" height="300" scrolling="yes" ' +\
'seamless="seamless"></iframe>\n'
graphs += ']]></ac:plain-text-body>\n'
@@ -177,7 +179,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
@@ -204,12 +206,13 @@
onpass="ONOS cli startup successful",
onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
appCheck = main.TRUE
@@ -636,6 +639,13 @@
str( sorted( onosIds ) ) )
if sorted( ids ) != sorted( intentIds ):
correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ correct = False
+ break
if correct:
break
else:
@@ -773,7 +783,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -785,9 +795,6 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
@@ -795,31 +802,37 @@
onfail="Intents have not been installed correctly, pings failed." )
main.step( "Check Intent state" )
- installedCheck = True
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ installedCheck = False
count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- # Print states
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
+ while not installedCheck and count < 40:
+ installedCheck = True
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
utilities.assert_equals( expect=True, actual=installedCheck,
onpass="Intents are all INSTALLED",
- onfail="Intents are not all in " +\
+ onfail="Intents are not all in " +
"INSTALLED state" )
main.step( "Check leadership of topics" )
@@ -970,9 +983,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- main.log.debug( CLIs[0].flows( jsonFormat=False ) )
-
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
+ # the wait is above
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -985,7 +999,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -1064,7 +1078,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1078,7 +1092,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1126,7 +1140,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1139,11 +1153,11 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1227,7 +1241,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1251,7 +1265,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1414,14 +1428,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1457,12 +1471,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1614,10 +1628,6 @@
main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
- time.sleep( 200 )
- main.log.debug( CLIs[0].nodes( jsonFormat=False ) )
- main.log.debug( CLIs[0].leaders( jsonFormat=False ) )
- main.log.debug( CLIs[0].partitions( jsonFormat=False ) )
def CASE7( self, main ):
"""
@@ -1669,7 +1679,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1683,7 +1693,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1757,7 +1767,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1770,7 +1780,7 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
@@ -1837,22 +1847,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1900,7 +1929,7 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
+ main.log.info( "No loss of dataplane connectivity" )
utilities.assert_equals(
expect=main.FALSE,
actual=LossInPings,
@@ -1920,17 +1949,17 @@
leaderList.append( leaderN )
if leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leaderN is None:
- main.log.report( cli.name +
+ main.log.error( cli.name +
" shows no leader for the election-app was" +
" elected after the old one died" )
leaderResult = main.FALSE
elif leaderN in restarted:
- main.log.report( cli.name + " shows " + str( leaderN ) +
+ main.log.error( cli.name + " shows " + str( leaderN ) +
" as leader for the election-app, but it " +
"was restarted" )
leaderResult = main.FALSE
@@ -2032,7 +2061,7 @@
for host in hosts[ controller ]:
if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ips on controller" +
+ "DEBUG:Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -2146,13 +2175,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2173,13 +2202,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2280,7 +2309,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2309,7 +2337,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2337,14 +2364,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2383,11 +2409,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2436,9 +2460,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2494,7 +2516,7 @@
for node in nodes:
print colors[ 'purple' ] + "Checking logs for errors on " + \
node.name + ":" + colors[ 'end' ]
- print main.ONOSbench.checkLogs( node.ip_address )
+ print main.ONOSbench.checkLogs( node.ip_address, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
@@ -2502,8 +2524,9 @@
try:
timerLog = open( main.logdir + "/Timers.csv", 'w')
# Overwrite with empty line and close
- timerLog.write( "Restart\n" )
- timerLog.write( str( main.restartTime ) )
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
timerLog.close()
except NameError, e:
main.log.exception(e)
@@ -2535,7 +2558,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2571,7 +2594,6 @@
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
main.step( "Check that each node shows the same leader" )
@@ -2621,14 +2643,14 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
@@ -2645,10 +2667,10 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
utilities.assert_equals(
@@ -2688,6 +2710,7 @@
"""
Install Distributed Primitives app
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -2717,7 +2740,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
- time.sleep (5 ) # To allow all nodes to activate
+ time.sleep( 5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2777,7 +2800,7 @@
# Check that counter incremented numController times
pCounterResults = True
for i in addedPValues:
- tmpResult = i in pCounters
+ tmpResult = i in pCounters
pCounterResults = pCounterResults and tmpResult
if not tmpResult:
main.log.error( str( i ) + " is not in partitioned "
@@ -2847,28 +2870,41 @@
main.step( "Counters we added have the correct values" )
correctResults = main.TRUE
for i in range( numControllers ):
- current = onosCounters[i]
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
try:
- pValue = current.get( pCounterName )
- iValue = current.get( iCounterName )
- if pValue == pCounterValue:
- main.log.info( "Partitioned counter value is correct" )
- else:
- main.log.error( "Partitioned counter value is incorrect," +
- " expected value: " + str( pCounterValue )
- + " current value: " + str( pValue ) )
- correctResults = main.FALSE
- if iValue == iCounterValue:
- main.log.info( "In memory counter value is correct" )
- else:
- main.log.error( "In memory counter value is incorrect, " +
- "expected value: " + str( iCounterValue ) +
- " current value: " + str( iValue ) )
- correctResults = main.FALSE
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
except AttributeError, e:
main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
"is not as expected" )
correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=correctResults,
onpass="Added counters are correct",
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
index e1421cd..f82aba3 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.params
@@ -25,6 +25,7 @@
<Git> False </Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
index e4658e7..d2a2ec1 100644
--- a/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
+++ b/TestON/tests/HATestNetworkPartition/HATestNetworkPartition.py
@@ -217,12 +217,13 @@
cliResults = cliResult1 and cliResult2 and cliResult3 and\
cliResult4 and cliResult5 and cliResult6 and cliResult7
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
case1Result = ( cleanInstallResult and packageResult and
cellResult and verifyResult and onosInstallResult
diff --git a/TestON/tests/HATestSanity/HATestSanity.params b/TestON/tests/HATestSanity/HATestSanity.params
index 2440ec6..9106d68 100644
--- a/TestON/tests/HATestSanity/HATestSanity.params
+++ b/TestON/tests/HATestSanity/HATestSanity.params
@@ -25,6 +25,7 @@
<Git> True </Git>
<branch> master </branch>
<num_controllers> 7 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestSanity/HATestSanity.py b/TestON/tests/HATestSanity/HATestSanity.py
index fe513ac..60c438e 100644
--- a/TestON/tests/HATestSanity/HATestSanity.py
+++ b/TestON/tests/HATestSanity/HATestSanity.py
@@ -47,7 +47,7 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS HA Sanity test - initialization" )
+ main.log.info( "ONOS HA Sanity test - initialization" )
main.case( "Setting up test environment" )
main.caseExplaination = "Setup the test environment including " +\
"installing ONOS, starting Mininet and ONOS" +\
@@ -95,15 +95,15 @@
verifyResult = main.ONOSbench.verifyCell()
# FIXME:this is short term fix
- main.log.report( "Removing raft logs" )
+ main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
- main.log.report( "Uninstalling ONOS" )
+ main.log.info( "Uninstalling ONOS" )
for node in nodes:
main.ONOSbench.onosUninstall( node.ip_address )
# Make sure ONOS is DEAD
- main.log.report( "Killing any ONOS processes" )
+ main.log.info( "Killing any ONOS processes" )
killResults = main.TRUE
for node in nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
@@ -126,15 +126,15 @@
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean install" )
cleanInstallResult = main.TRUE
- if gitPullResult == main.TRUE:
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
utilities.assert_equals( expect=main.TRUE,
actual=cleanInstallResult,
onpass="MCI successful",
@@ -179,7 +179,7 @@
for node in nodes:
started = main.ONOSbench.isup( node.ip_address )
if not started:
- main.log.report( node.name + " didn't start!" )
+ main.log.error( node.name + " didn't start!" )
main.ONOSbench.onosStop( node.ip_address )
main.ONOSbench.onosStart( node.ip_address )
onosIsupResult = onosIsupResult and started
@@ -206,12 +206,13 @@
onpass="ONOS cli startup successful",
onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
appCheck = main.TRUE
@@ -294,13 +295,12 @@
"not in the list of controllers s" +
str( i ) + " is connecting to." )
mastershipCheck = main.FALSE
- if mastershipCheck == main.TRUE:
- main.log.report( "Switch mastership assigned correctly" )
utilities.assert_equals(
expect=main.TRUE,
actual=mastershipCheck,
onpass="Switch mastership assigned correctly",
onfail="Switches not assigned correctly to controllers" )
+ # FIXME: Check topo here
main.step( "Assign mastership of switches to specific controllers" )
# Manually assign mastership to the controller we want
@@ -634,7 +634,7 @@
main.log.error( "Error in pushing host intents to ONOS" )
main.step( "Intent Anti-Entropy dispersion" )
- for i in range(100):
+ for i in range(80):
correct = True
main.log.info( "Submitted intents: " + str( sorted( intentIds ) ) )
for cli in CLIs:
@@ -644,11 +644,19 @@
main.log.debug( "Intents in " + cli.name + ": " +
str( sorted( onosIds ) ) )
if sorted( ids ) != sorted( intentIds ):
+ main.log.debug( "Set of intent IDs doesn't match" )
correct = False
+ break
+ else:
+ intents = json.loads( cli.intents() )
+ for intent in intents:
+ if intent[ 'state' ] != "INSTALLED":
+ main.log.warn( "Intent " + intent[ 'id' ] +
+ " is " + intent[ 'state' ] )
+ correct = False
+ break
if correct:
break
- else:
- time.sleep(1)
if not intentStop:
intentStop = time.time()
global gossipTime
@@ -782,7 +790,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -794,9 +802,6 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.report(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
@@ -804,31 +809,37 @@
onfail="Intents have not been installed correctly, pings failed." )
main.step( "Check Intent state" )
- installedCheck = True
- # Print the intent states
- intents = main.ONOScli1.intents()
- intentStates = []
- main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ installedCheck = False
count = 0
- # Iter through intents of a node
- try:
- for intent in json.loads( intents ):
- state = intent.get( 'state', None )
- if "INSTALLED" not in state:
- installedCheck = False
- intentId = intent.get( 'id', None )
- intentStates.append( ( intentId, state ) )
- except ( ValueError, TypeError ):
- main.log.exception( "Error parsing intents." )
- # Print states
- intentStates.sort()
- for i, s in intentStates:
- count += 1
- main.log.info( "%-6s%-15s%-15s" %
- ( str( count ), str( i ), str( s ) ) )
+ while not installedCheck and count < 40:
+ installedCheck = True
+ # Print the intent states
+ intents = main.ONOScli1.intents()
+ intentStates = []
+ main.log.info( "%-6s%-15s%-15s" % ( 'Count', 'ID', 'State' ) )
+ count = 0
+ # Iter through intents of a node
+ try:
+ for intent in json.loads( intents ):
+ state = intent.get( 'state', None )
+ if "INSTALLED" not in state:
+ installedCheck = False
+ intentId = intent.get( 'id', None )
+ intentStates.append( ( intentId, state ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Error parsing intents." )
+ # Print states
+ intentStates.sort()
+ for i, s in intentStates:
+ count += 1
+ main.log.info( "%-6s%-15s%-15s" %
+ ( str( count ), str( i ), str( s ) ) )
+ if not installedCheck:
+ time.sleep( 1 )
+ count += 1
utilities.assert_equals( expect=True, actual=installedCheck,
onpass="Intents are all INSTALLED",
- onfail="Intents are not all in " +\
+ onfail="Intents are not all in " +
"INSTALLED state" )
main.step( "Check leadership of topics" )
@@ -979,9 +990,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- main.log.debug( CLIs[0].flows( jsonFormat=False ) )
-
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
+ # the wait is above
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -994,7 +1006,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -1011,15 +1023,6 @@
actual=PingResult,
onpass="Intents have been installed correctly and pings work",
onfail="Intents have not been installed correctly, pings failed." )
-
-
-
- '''
- #DEBUG
- if PingResult == main.FALSE:
- import time
- time.sleep( 100000 )
- '''
def CASE5( self, main ):
"""
@@ -1082,7 +1085,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1096,7 +1099,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1144,7 +1147,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1157,11 +1160,11 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
- main.log.report( "Intents not consistent" )
+ main.log.error( "Intents not consistent" )
utilities.assert_equals(
expect=True,
actual=consistentIntents,
@@ -1179,22 +1182,27 @@
for n in range( numControllers ):
title += " " * 10 + "ONOS" + str( n + 1 )
main.log.warn( title )
- # get all intent keys in the cluster
keys = []
- for nodeStr in ONOSIntents:
- node = json.loads( nodeStr )
- for intent in node:
- keys.append( intent.get( 'id' ) )
- keys = set( keys )
- for key in keys:
- row = "%-13s" % key
+ try:
+ # Get the set of all intent keys
for nodeStr in ONOSIntents:
node = json.loads( nodeStr )
for intent in node:
- if intent.get( 'id', "Error" ) == key:
- row += "%-15s" % intent.get( 'state' )
- main.log.warn( row )
- # End table view
+ keys.append( intent.get( 'id' ) )
+ keys = set( keys )
+ # For each intent key, print the state on each node
+ for key in keys:
+ row = "%-13s" % key
+ for nodeStr in ONOSIntents:
+ node = json.loads( nodeStr )
+ for intent in node:
+ if intent.get( 'id', "Error" ) == key:
+ row += "%-15s" % intent.get( 'state' )
+ main.log.warn( row )
+ # End of intent state table
+ except ValueError as e:
+ main.log.exception( e )
+ main.log.debug( "nodeStr was: " + repr( nodeStr ) )
if intentsResults and not consistentIntents:
# print the json objects
@@ -1245,7 +1253,7 @@
for i in range( numControllers ):
num = str( i + 1 )
if not ONOSFlows[ i ] or "Error" in ONOSFlows[ i ]:
- main.log.report( "Error in getting ONOS" + num + " flows" )
+ main.log.error( "Error in getting ONOS" + num + " flows" )
main.log.warn( "ONOS" + num + " flows response: " +
repr( ONOSFlows[ i ] ) )
flowsResults = False
@@ -1269,7 +1277,7 @@
main.step( "Check for consistency in Flows from each controller" )
tmp = [ len( i ) == len( ONOSFlowsJson[ 0 ] ) for i in ONOSFlowsJson ]
if all( tmp ):
- main.log.report( "Flow count is consistent across all ONOS nodes" )
+ main.log.info( "Flow count is consistent across all ONOS nodes" )
else:
consistentFlows = False
utilities.assert_equals(
@@ -1432,14 +1440,14 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" +
+ main.log.error( "hosts from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1475,12 +1483,12 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" + controllerStr +
+ main.log.error( "clusters from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -1632,7 +1640,7 @@
for i in range( numControllers ):
if not ONOSMastership[i] or "Error" in ONOSMastership[i]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" roles" )
main.log.warn(
"ONOS" + str( i + 1 ) + " mastership response: " +
@@ -1646,7 +1654,7 @@
main.step( "Check for consistency in roles from each controller" )
if all([ i == ONOSMastership[ 0 ] for i in ONOSMastership ] ):
- main.log.report(
+ main.log.info(
"Switch roles are consistent across all ONOS nodes" )
else:
consistentMastership = False
@@ -1720,7 +1728,7 @@
for i in range( numControllers ):
if not ONOSIntents[ i ] or "Error" in ONOSIntents[ i ]:
- main.log.report( "Error in getting ONOS" + str( i + 1 ) +
+ main.log.error( "Error in getting ONOS" + str( i + 1 ) +
" intents" )
main.log.warn( "ONOS" + str( i + 1 ) + " intents response: " +
repr( ONOSIntents[ i ] ) )
@@ -1733,7 +1741,7 @@
main.step( "Check for consistency in Intents from each controller" )
if all([ sorted( i ) == sorted( ONOSIntents[ 0 ] ) for i in ONOSIntents ] ):
- main.log.report( "Intents are consistent across all ONOS " +
+ main.log.info( "Intents are consistent across all ONOS " +
"nodes" )
else:
consistentIntents = False
@@ -1800,22 +1808,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
+ sameIntents = main.FALSE
if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS intents: " )
- main.log.warn( json.dumps( json.loads( ONOSIntents[ 0 ] ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) ) )
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
except ( ValueError, TypeError ):
main.log.exception( "Exception printing intents" )
- main.log.warn( repr( ONOSIntents[0] ) )
- sameIntents = main.FALSE
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1863,7 +1890,7 @@
main.log.info( "There are multiple mininet process running" )
elif LossInPings == main.FALSE:
main.log.info( "No Loss in the pings" )
- main.log.report( "No loss of dataplane connectivity" )
+ main.log.info( "No loss of dataplane connectivity" )
utilities.assert_equals(
expect=main.FALSE,
actual=LossInPings,
@@ -1886,13 +1913,13 @@
pass
elif leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( cli.name + " sees " + str( leaderN ) +
+ main.log.error( cli.name + " sees " + str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
utilities.assert_equals(
@@ -1988,7 +2015,7 @@
for host in hosts[ controller ]:
if host is None or host.get( 'ipAddresses', [] ) == []:
main.log.error(
- "DEBUG:Error with host ips on controller" +
+ "DEBUG:Error with host ipAddresses on controller" +
controllerStr + ": " + str( host ) )
ipResult = main.FALSE
ports = []
@@ -2102,13 +2129,13 @@
if hosts[ controller ] == hosts[ 0 ]:
continue
else: # hosts not consistent
- main.log.report( "hosts from ONOS" + controllerStr +
+ main.log.error( "hosts from ONOS" + controllerStr +
" is inconsistent with ONOS1" )
main.log.warn( repr( hosts[ controller ] ) )
consistentHostsResult = main.FALSE
else:
- main.log.report( "Error in getting ONOS hosts from ONOS" +
+ main.log.error( "Error in getting ONOS hosts from ONOS" +
controllerStr )
consistentHostsResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2129,13 +2156,13 @@
if clusters[ controller ] == clusters[ 0 ]:
continue
else: # clusters not consistent
- main.log.report( "clusters from ONOS" +
+ main.log.error( "clusters from ONOS" +
controllerStr +
" is inconsistent with ONOS1" )
consistentClustersResult = main.FALSE
else:
- main.log.report( "Error in getting dataplane clusters " +
+ main.log.error( "Error in getting dataplane clusters " +
"from ONOS" + controllerStr )
consistentClustersResult = main.FALSE
main.log.warn( "ONOS" + controllerStr +
@@ -2236,7 +2263,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -2265,7 +2291,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -2293,14 +2318,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -2339,11 +2363,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -2392,9 +2414,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -2458,8 +2478,9 @@
try:
timerLog = open( main.logdir + "/Timers.csv", 'w')
# Overwrite with empty line and close
- timerLog.write( "Gossip Intents\n" )
- timerLog.write( str( gossipTime ) )
+ labels = "Gossip Intents"
+ data = str( gossipTime )
+ timerLog.write( labels + "\n" + data )
timerLog.close()
except NameError, e:
main.log.exception(e)
@@ -2491,7 +2512,7 @@
for cli in CLIs:
leader = cli.electionTestLeader()
if leader is None or leader == main.FALSE:
- main.log.report( cli.name + ": Leader for the election app " +
+ main.log.error( cli.name + ": Leader for the election app " +
"should be an ONOS node, instead got '" +
str( leader ) + "'" )
leaderResult = main.FALSE
@@ -2527,7 +2548,6 @@
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
main.step( "Check that each node shows the same leader" )
@@ -2577,14 +2597,14 @@
leaderN = cli.electionTestLeader()
leaderList.append( leaderN )
if leaderN == leader:
- main.log.report( cli.name + " still sees " + str( leader ) +
+ main.log.error( cli.name + " still sees " + str( leader ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, " +
"check the error logs" )
leaderResult = main.FALSE
@@ -2601,10 +2621,10 @@
"' as the leader" )
consistentLeader = main.TRUE
else:
- main.log.report(
+ main.log.error(
"Inconsistent responses for leader of Election-app:" )
for n in range( len( leaderList ) ):
- main.log.report( "ONOS" + str( n + 1 ) + " response: " +
+ main.log.error( "ONOS" + str( n + 1 ) + " response: " +
str( leaderList[ n ] ) )
leaderResult = leaderResult and consistentLeader
utilities.assert_equals(
@@ -2644,6 +2664,7 @@
"""
Install Distributed Primitives app
"""
+ import time
assert numControllers, "numControllers not defined"
assert main, "main not defined"
assert utilities.assert_equals, "utilities.assert_equals not defined"
@@ -2673,7 +2694,7 @@
actual=appResults,
onpass="Primitives app activated",
onfail="Primitives app not activated" )
- time.sleep (5 ) # To allow all nodes to activate
+ time.sleep( 5 ) # To allow all nodes to activate
def CASE17( self, main ):
"""
@@ -2733,7 +2754,7 @@
# Check that counter incremented numController times
pCounterResults = True
for i in addedPValues:
- tmpResult = i in pCounters
+ tmpResult = i in pCounters
pCounterResults = pCounterResults and tmpResult
if not tmpResult:
main.log.error( str( i ) + " is not in partitioned "
@@ -2803,28 +2824,41 @@
main.step( "Counters we added have the correct values" )
correctResults = main.TRUE
for i in range( numControllers ):
- current = onosCounters[i]
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
try:
- pValue = current.get( pCounterName )
- iValue = current.get( iCounterName )
- if pValue == pCounterValue:
- main.log.info( "Partitioned counter value is correct" )
- else:
- main.log.error( "Partitioned counter value is incorrect," +
- " expected value: " + str( pCounterValue )
- + " current value: " + str( pValue ) )
- correctResults = main.FALSE
- if iValue == iCounterValue:
- main.log.info( "In memory counter value is correct" )
- else:
- main.log.error( "In memory counter value is incorrect, " +
- "expected value: " + str( iCounterValue ) +
- " current value: " + str( iValue ) )
- correctResults = main.FALSE
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
except AttributeError, e:
main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
"is not as expected" )
correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=correctResults,
onpass="Added counters are correct",
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
index f057592..108ab88 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.params
@@ -23,6 +23,7 @@
<Git>False</Git>
<branch> master </branch>
<num_controllers> 1 </num_controllers>
+ <tcpdump> False </tcpdump>
<CTRL>
<ip1>10.128.30.11</ip1>
diff --git a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
index 7194a3b..8f3c604 100644
--- a/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
+++ b/TestON/tests/HATestSingleInstanceRestart/HATestSingleInstanceRestart.py
@@ -46,7 +46,7 @@
start cli sessions
start tcpdump
"""
- main.log.report( "ONOS Single node cluster restart " +
+ main.log.info( "ONOS Single node cluster restart " +
"HA test - initialization" )
main.case( "Setting up test environment" )
main.caseExplaination = "Setup the test environment including " +\
@@ -126,15 +126,15 @@
utilities.assert_lesser( expect=0, actual=gitPullResult,
onpass="Git pull successful",
onfail="Git pull failed" )
- else:
- main.log.warn( "Did not pull new code so skipping mvn " +
- "clean install" )
main.ONOSbench.getVersion( report=True )
main.step( "Using mvn clean install" )
cleanInstallResult = main.TRUE
- if gitPullResult == main.TRUE:
+ if PULLCODE and gitPullResult == main.TRUE:
cleanInstallResult = main.ONOSbench.cleanInstall()
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
utilities.assert_equals( expect=main.TRUE,
actual=cleanInstallResult,
onpass="MCI successful",
@@ -145,10 +145,12 @@
# Plot Name = Plot-HA, only can be used if multiple plots
# index = The number of the graph under plot name
job = "HASingleInstanceRestart"
+ plotName = "Plot-HA"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
- '/plot/getPlot?index=0&width=500&height=300"' +\
+ '/plot/' + plotName + '/getPlot?index=0' +\
+ '&width=500&height=300"' +\
'noborder="0" width="500" height="300" scrolling="yes" ' +\
'seamless="seamless"></iframe>\n'
graphs += ']]></ac:plain-text-body>\n'
@@ -185,12 +187,13 @@
onpass="ONOS cli startup successful",
onfail="ONOS cli startup failed" )
- main.step( "Start Packet Capture MN" )
- main.Mininet2.startTcpdump(
- str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
- + "-MN.pcap",
- intf=main.params[ 'MNtcpdump' ][ 'intf' ],
- port=main.params[ 'MNtcpdump' ][ 'port' ] )
+ if main.params[ 'tcpdump' ].lower() == "true":
+ main.step( "Start Packet Capture MN" )
+ main.Mininet2.startTcpdump(
+ str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
+ + "-MN.pcap",
+ intf=main.params[ 'MNtcpdump' ][ 'intf' ],
+ port=main.params[ 'MNtcpdump' ][ 'port' ] )
main.step( "App Ids check" )
appCheck = main.ONOScli1.appToIDCheck()
@@ -681,7 +684,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -693,9 +696,6 @@
separators=( ',', ': ' ) ) )
except ( ValueError, TypeError ):
main.log.warn( repr( tmpIntents ) )
- if PingResult == main.TRUE:
- main.log.info(
- "Intents have been installed correctly and verified by pings" )
utilities.assert_equals(
expect=main.TRUE,
actual=PingResult,
@@ -727,7 +727,7 @@
( str( count ), str( i ), str( s ) ) )
utilities.assert_equals( expect=True, actual=installedCheck,
onpass="Intents are all INSTALLED",
- onfail="Intents are not all in " +\
+ onfail="Intents are not all in " +
"INSTALLED state" )
main.step( "Check leadership of topics" )
@@ -878,9 +878,10 @@
except ( ValueError, TypeError ):
main.log.exception( "Error parsing pending map" )
main.log.error( repr( pendingMap ) )
- main.log.debug( main.ONOScli1.flows( jsonFormat=False ) )
-
+ # Print flowrules
+ main.log.debug( CLIs[0].flows( jsonFormat=False ) )
main.step( "Wait a minute then ping again" )
+ # the wait is above
PingResult = main.TRUE
for i in range( 8, 18 ):
ping = main.Mininet1.pingHost( src="h" + str( i ),
@@ -893,7 +894,7 @@
main.log.info( "Ping test passed!" )
# Don't set PingResult or you'd override failures
if PingResult == main.FALSE:
- main.log.report(
+ main.log.error(
"Intents have not been installed correctly, pings failed." )
# TODO: pretty print
main.log.warn( "ONOS1 intents: " )
@@ -943,7 +944,7 @@
ONOS1Mastership = main.ONOScli1.roles()
# TODO: Make this a meaningful check
if "Error" in ONOS1Mastership or not ONOS1Mastership:
- main.log.report( "Error in getting ONOS roles" )
+ main.log.error( "Error in getting ONOS roles" )
main.log.warn(
"ONOS1 mastership response: " +
repr( ONOS1Mastership ) )
@@ -958,7 +959,7 @@
ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
intentCheck = main.FALSE
if "Error" in ONOS1Intents or not ONOS1Intents:
- main.log.report( "Error in getting ONOS intents" )
+ main.log.error( "Error in getting ONOS intents" )
main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
else:
intentCheck = main.TRUE
@@ -969,7 +970,7 @@
flowCheck = main.FALSE
ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
if "Error" in ONOS1Flows or not ONOS1Flows:
- main.log.report( "Error in getting ONOS flows" )
+ main.log.error( "Error in getting ONOS flows" )
main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
else:
# TODO: Do a better check, maybe compare flows on switches?
@@ -1121,7 +1122,6 @@
main.log.error( "iCounterValue not defined, setting to 0" )
iCounterValue = 0
- main.log.report( "Restart ONOS node" )
main.case( "Restart ONOS node" )
main.caseExplaination = "Killing ONOS process and restart cli " +\
"sessions once onos is up."
@@ -1154,6 +1154,9 @@
if elapsed:
main.log.info( "ESTIMATE: ONOS took %s seconds to restart" %
str( elapsed ) )
+ main.restartTime = elapsed
+ else:
+ main.restartTime = -1
time.sleep( 5 )
# rerun on election apps
main.ONOScli1.electionTestRun()
@@ -1180,14 +1183,12 @@
ONOS1Mastership = main.ONOScli1.roles()
# FIXME: Refactor this whole case for single instance
if "Error" in ONOS1Mastership or not ONOS1Mastership:
- main.log.report( "Error in getting ONOS mastership" )
+ main.log.error( "Error in getting ONOS mastership" )
main.log.warn( "ONOS1 mastership response: " +
repr( ONOS1Mastership ) )
consistentMastership = main.FALSE
else:
consistentMastership = main.TRUE
- main.log.report(
- "Switch roles are consistent across all ONOS nodes" )
utilities.assert_equals(
expect=main.TRUE,
actual=consistentMastership,
@@ -1224,11 +1225,11 @@
ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
intentCheck = main.FALSE
if "Error" in ONOS1Intents or not ONOS1Intents:
- main.log.report( "Error in getting ONOS intents" )
+ main.log.error( "Error in getting ONOS intents" )
main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
else:
intentCheck = main.TRUE
- main.log.report( "Intents are consistent across all ONOS nodes" )
+ main.log.error( "Intents are consistent across all ONOS nodes" )
utilities.assert_equals(
expect=main.TRUE,
actual=intentCheck,
@@ -1253,21 +1254,41 @@
main.step( "Compare current intents with intents before the failure" )
# NOTE: this requires case 5 to pass for intentState to be set.
# maybe we should stop the test if that fails?
- sameIntents = main.TRUE
- if intentState and intentState == ONOS1Intents:
+ sameIntents = main.FALSE
+ if intentState and intentState == ONOSIntents[ 0 ]:
sameIntents = main.TRUE
main.log.info( "Intents are consistent with before failure" )
# TODO: possibly the states have changed? we may need to figure out
# what the acceptable states are
- else:
+ elif len( intentState ) == len( ONOSIntents[ 0 ] ):
+ sameIntents = main.TRUE
try:
- main.log.warn( "ONOS1 intents: " )
- print json.dumps( json.loads( ONOS1Intents ),
- sort_keys=True, indent=4,
- separators=( ',', ': ' ) )
- except Exception:
- pass
- sameIntents = main.FALSE
+ before = json.loads( intentState )
+ after = json.loads( ONOSIntents[ 0 ] )
+ for intent in before:
+ if intent not in after:
+ sameIntents = main.FALSE
+ main.log.debug( "Intent is not currently in ONOS " +\
+ "(at least in the same form):" )
+ main.log.debug( json.dumps( intent ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
+ if sameIntents == main.FALSE:
+ try:
+ main.log.debug( "ONOS intents before: " )
+ main.log.debug( json.dumps( json.loads( intentState ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ main.log.debug( "Current ONOS intents: " )
+ main.log.debug( json.dumps( json.loads( ONOSIntents[ 0 ] ),
+ sort_keys=True, indent=4,
+ separators=( ',', ': ' ) ) )
+ except ( ValueError, TypeError ):
+ main.log.exception( "Exception printing intents" )
+ main.log.debug( repr( ONOSIntents[0] ) )
+ main.log.debug( repr( intentState ) )
utilities.assert_equals(
expect=main.TRUE,
actual=sameIntents,
@@ -1313,13 +1334,13 @@
pass
elif leaderN == main.FALSE:
# error in response
- main.log.report( "Something is wrong with " +
+ main.log.error( "Something is wrong with " +
"electionTestLeader function, check the" +
" error logs" )
leaderResult = main.FALSE
elif leader != leaderN:
leaderResult = main.FALSE
- main.log.report( "ONOS" + str( controller ) + " sees " +
+ main.log.error( "ONOS" + str( controller ) + " sees " +
str( leaderN ) +
" as the leader of the election app. " +
"Leader should be " + str( leader ) )
@@ -1504,7 +1525,6 @@
description = "Turn off a link to ensure that Link Discovery " +\
"is working properly"
- main.log.report( description )
main.case( description )
main.step( "Kill Link between s3 and s28" )
@@ -1531,7 +1551,6 @@
description = "Restore a link to ensure that Link Discovery is " + \
"working properly"
- main.log.report( description )
main.case( description )
main.step( "Bring link between s3 and s28 back up" )
@@ -1557,14 +1576,13 @@
switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
description = "Killing a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
switch = main.params[ 'kill' ][ 'switch' ]
switchDPID = main.params[ 'kill' ][ 'dpid' ]
# TODO: Make this switch parameterizable
main.step( "Kill " + switch )
- main.log.report( "Deleting " + switch )
+ main.log.info( "Deleting " + switch )
main.Mininet1.delSwitch( switch )
main.log.info( "Waiting " + str( switchSleep ) +
" seconds for switch down to be discovered" )
@@ -1601,11 +1619,9 @@
switchDPID = main.params[ 'kill' ][ 'dpid' ]
links = main.params[ 'kill' ][ 'links' ].split()
description = "Adding a switch to ensure it is discovered correctly"
- main.log.report( description )
main.case( description )
main.step( "Add back " + switch )
- main.log.report( "Adding back " + switch )
main.Mininet1.addSwitch( switch, dpid=switchDPID )
for peer in links:
main.Mininet1.addLink( switch, peer )
@@ -1638,9 +1654,7 @@
colors = { 'cyan': '\033[96m', 'purple': '\033[95m',
'blue': '\033[94m', 'green': '\033[92m',
'yellow': '\033[93m', 'red': '\033[91m', 'end': '\033[0m' }
- description = "Test Cleanup"
- main.log.report( description )
- main.case( description )
+ main.case( "Test Cleanup" )
main.step( "Killing tcpdumps" )
main.Mininet2.stopTcpdump()
@@ -1691,11 +1705,21 @@
main.step( "Checking ONOS Logs for errors" )
print colors[ 'purple' ] + "Checking logs for errors on ONOS1:" + \
colors[ 'end' ]
- print main.ONOSbench.checkLogs( ONOS1Ip )
+ print main.ONOSbench.checkLogs( ONOS1Ip, restart=True )
main.step( "Packing and rotating pcap archives" )
os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
+ try:
+ timerLog = open( main.logdir + "/Timers.csv", 'w')
+ # Overwrite with empty line and close
+ labels = "Gossip Intents, Restart"
+ data = str( gossipTime ) + ", " + str( main.restartTime )
+ timerLog.write( labels + "\n" + data )
+ timerLog.close()
+ except NameError, e:
+ main.log.exception(e)
+
def CASE14( self, main ):
"""
start election app on all onos nodes
@@ -1754,7 +1778,6 @@
assert utilities.assert_equals, "utilities.assert_equals not defined"
leaderResult = main.TRUE
description = "Check that Leadership Election is still functional"
- main.log.report( description )
main.case( description )
main.step( "Find current leader and withdraw" )
leader = main.ONOScli1.electionTestLeader()
@@ -1763,7 +1786,7 @@
if leader == ONOS1Ip:
oldLeader = getattr( main, "ONOScli1" )
elif leader is None or leader == main.FALSE:
- main.log.report(
+ main.log.error(
"Leader for the election app should be an ONOS node," +
"instead got '" + str( leader ) + "'" )
leaderResult = main.FALSE
@@ -1783,14 +1806,14 @@
main.step( "Make sure new leader is elected" )
leaderN = main.ONOScli1.electionTestLeader()
if leaderN == leader:
- main.log.report( "ONOS still sees " + str( leaderN ) +
+ main.log.error( "ONOS still sees " + str( leaderN ) +
" as leader after they withdrew" )
leaderResult = main.FALSE
elif leaderN == main.FALSE:
# error in response
# TODO: add check for "Command not found:" in the driver, this
# means the app isn't loaded
- main.log.report( "Something is wrong with electionTestLeader " +
+ main.log.error( "Something is wrong with electionTestLeader " +
"function, check the error logs" )
leaderResult = main.FALSE
elif leaderN is None:
@@ -1868,6 +1891,7 @@
"""
Check for basic functionality with distributed primitives
"""
+ import json
# Make sure variables are defined/set
assert numControllers, "numControllers not defined"
assert main, "main not defined"
@@ -1922,7 +1946,7 @@
# Check that counter incremented numController times
pCounterResults = True
for i in addedPValues:
- tmpResult = i in pCounters
+ tmpResult = i in pCounters
pCounterResults = pCounterResults and tmpResult
if not tmpResult:
main.log.error( str( i ) + " is not in partitioned "
@@ -1992,28 +2016,41 @@
main.step( "Counters we added have the correct values" )
correctResults = main.TRUE
for i in range( numControllers ):
- current = onosCounters[i]
+ current = json.loads( onosCounters[i] )
+ pValue = None
+ iValue = None
try:
- pValue = current.get( pCounterName )
- iValue = current.get( iCounterName )
- if pValue == pCounterValue:
- main.log.info( "Partitioned counter value is correct" )
- else:
- main.log.error( "Partitioned counter value is incorrect," +
- " expected value: " + str( pCounterValue )
- + " current value: " + str( pValue ) )
- correctResults = main.FALSE
- if iValue == iCounterValue:
- main.log.info( "In memory counter value is correct" )
- else:
- main.log.error( "In memory counter value is incorrect, " +
- "expected value: " + str( iCounterValue ) +
- " current value: " + str( iValue ) )
- correctResults = main.FALSE
+ for database in current:
+ partitioned = database.get( 'partitionedDatabaseCounters' )
+ if partitioned:
+ for value in partitioned:
+ if value.get( 'name' ) == pCounterName:
+ pValue = value.get( 'value' )
+ break
+ inMemory = database.get( 'inMemoryDatabaseCounters' )
+ if inMemory:
+ for value in inMemory:
+ if value.get( 'name' ) == iCounterName:
+ iValue = value.get( 'value' )
+ break
except AttributeError, e:
main.log.error( "ONOS" + str( i + 1 ) + " counters result " +
"is not as expected" )
correctResults = main.FALSE
+ if pValue == pCounterValue:
+ main.log.info( "Partitioned counter value is correct" )
+ else:
+ main.log.error( "Partitioned counter value is incorrect," +
+ " expected value: " + str( pCounterValue )
+ + " current value: " + str( pValue ) )
+ correctResults = main.FALSE
+ if iValue == iCounterValue:
+ main.log.info( "In memory counter value is correct" )
+ else:
+ main.log.error( "In memory counter value is incorrect, " +
+ "expected value: " + str( iCounterValue ) +
+ " current value: " + str( iValue ) )
+ correctResults = main.FALSE
utilities.assert_equals( expect=main.TRUE,
actual=correctResults,
onpass="Added counters are correct",
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.params b/TestON/tests/SdnIpTest/SdnIpTest.params
index 1fb33df..c84af17 100755
--- a/TestON/tests/SdnIpTest/SdnIpTest.params
+++ b/TestON/tests/SdnIpTest/SdnIpTest.params
@@ -16,7 +16,7 @@
<GIT>
<autoPull>on</autoPull>
<branch1>master</branch1>
- <branch2>onos-1.1</branch2>
+ <branch2>onos-1.2</branch2>
</GIT>
<JSON>
diff --git a/TestON/tests/SdnIpTest/SdnIpTest.py b/TestON/tests/SdnIpTest/SdnIpTest.py
index 3a766dd..43bf580 100644
--- a/TestON/tests/SdnIpTest/SdnIpTest.py
+++ b/TestON/tests/SdnIpTest/SdnIpTest.py
@@ -51,8 +51,6 @@
cleanInstallResult = main.ONOSbench.cleanInstall( mciTimeout= 1000 )
main.ONOSbench.getVersion( report=True )
- #cellResult = main.ONOSbench.setCell( cellName )
- #verifyResult = main.ONOSbench.verifyCell()
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.onosPackage( opTimeout=500 )
@@ -61,7 +59,6 @@
node=ONOS1Ip )
main.step( "Checking if ONOS is up yet" )
- #time.sleep( 300 )
for i in range( 2 ):
onos1Isup = main.ONOSbench.isup( ONOS1Ip, timeout=420 )
if onos1Isup:
@@ -148,8 +145,271 @@
listResult = main.ONOScli.devices( jsonFormat=False )
main.log.info( listResult )
time.sleep( 10 )
- main.log.info( "Installing sdn-ip feature" )
- main.ONOScli.featureInstall( "onos-app-sdnip" )
+ main.log.info( "Activate sdn-ip application" )
+ main.ONOScli.activateApp( "org.onosproject.sdnip" )
+ time.sleep( 10 )
+ main.step( "Login all BGP peers and add routes into peers" )
+
+ main.log.info( "Login Quagga CLI on host3" )
+ main.QuaggaCliHost3.loginQuagga( "1.168.30.2" )
+ main.log.info( "Enter configuration model of Quagga CLI on host3" )
+ main.QuaggaCliHost3.enterConfig( 64514 )
+ main.log.info( "Add routes to Quagga on host3" )
+ main.QuaggaCliHost3.addRoutes( prefixesHost3, 1 )
+
+ main.log.info( "Login Quagga CLI on host4" )
+ main.QuaggaCliHost4.loginQuagga( "1.168.30.3" )
+ main.log.info( "Enter configuration model of Quagga CLI on host4" )
+ main.QuaggaCliHost4.enterConfig( 64516 )
+ main.log.info( "Add routes to Quagga on host4" )
+ main.QuaggaCliHost4.addRoutes( prefixesHost4, 1 )
+
+ main.log.info( "Login Quagga CLI on host5" )
+ main.QuaggaCliHost5.loginQuagga( "1.168.30.5" )
+ main.log.info( "Enter configuration model of Quagga CLI on host5" )
+ main.QuaggaCliHost5.enterConfig( 64521 )
+ main.log.info( "Add routes to Quagga on host5" )
+ main.QuaggaCliHost5.addRoutes( prefixesHost5, 1 )
+
+ for i in range( 101, 201 ):
+ prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
+ main.log.info( prefixesHostX )
+ for prefix in prefixesHostX:
+ allRoutesExpected.append( prefix + "/" + "192.168.40."
+ + str( i - 100 ) )
+
+ routeIntentsExpectedHostX = \
+ main.QuaggaCliHost.generateExpectedOnePeerRouteIntents(
+ prefixesHostX, "192.168.40." + str( i - 100 ),
+ "00:00:%02d:00:00:90" % ( i - 101 ), SDNIPJSONFILEPATH )
+ routeIntentsExpected = routeIntentsExpected + \
+ routeIntentsExpectedHostX
+
+ main.log.info( "Login Quagga CLI on host" + str( i ) )
+ QuaggaCliHostX = getattr( main, ( 'QuaggaCliHost' + str( i ) ) )
+ QuaggaCliHostX.loginQuagga( "1.168.30." + str( i ) )
+ main.log.info(
+ "Enter configuration model of Quagga CLI on host" + str( i ) )
+ QuaggaCliHostX.enterConfig( 65000 + i - 100 )
+ main.log.info( "Add routes to Quagga on host" + str( i ) )
+ QuaggaCliHostX.addRoutes( prefixesHostX, 1 )
+
+ time.sleep( 60 )
+ # get routes inside SDN-IP
+ getRoutesResult = main.ONOScli.routes( jsonFormat=True )
+
+ allRoutesActual = \
+ main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+
+ allRoutesStrExpected = str( sorted( allRoutesExpected ) )
+ allRoutesStrActual = str( allRoutesActual ).replace( 'u', "" )
+ main.step( "Check routes installed" )
+ main.log.info( "Routes expected:" )
+ main.log.info( allRoutesStrExpected )
+ main.log.info( "Routes get from ONOS CLI:" )
+ main.log.info( allRoutesStrActual )
+ utilities.assertEquals(
+ expect=allRoutesStrExpected, actual=allRoutesStrActual,
+ onpass="***Routes in SDN-IP are correct!***",
+ onfail="***Routes in SDN-IP are wrong!***" )
+ if( eq( allRoutesStrExpected, allRoutesStrActual ) ):
+ main.log.report(
+ "***Routes in SDN-IP after adding routes are correct!***" )
+ else:
+ main.log.report(
+ "***Routes in SDN-IP after adding routes are wrong!***" )
+
+ time.sleep( 20 )
+ getIntentsResult = main.ONOScli.intents( jsonFormat=True )
+
+ main.step( "Check MultiPointToSinglePointIntent intents installed" )
+ # routeIntentsExpected are generated when generating routes
+ # get route intents from ONOS CLI
+ routeIntentsActualNum = \
+ main.QuaggaCliHost3.extractActualRouteIntentNum( getIntentsResult )
+ routeIntentsExpectedNum = 1030
+ main.log.info( "MultiPointToSinglePoint Intent Num expected is:" )
+ main.log.info( routeIntentsExpectedNum )
+ main.log.info( "MultiPointToSinglePoint Intent NUM Actual is:" )
+ main.log.info( routeIntentsActualNum )
+ utilities.assertEquals(
+ expect=True,
+ actual=eq( routeIntentsExpectedNum, routeIntentsActualNum ),
+ onpass="***MultiPointToSinglePoint Intent Num in SDN-IP is \
+ correct!***",
+ onfail="***MultiPointToSinglePoint Intent Num in SDN-IP is \
+ wrong!***" )
+
+ if( eq( routeIntentsExpectedNum, routeIntentsActualNum ) ):
+ main.log.report( "***MultiPointToSinglePoint Intents before \
+ deleting routes correct!***" )
+ else:
+ main.log.report( "***MultiPointToSinglePoint Intents before \
+ deleting routes wrong!***" )
+
+ main.step( "Check BGP PointToPointIntent intents installed" )
+
+ bgpIntentsActualNum = \
+ main.QuaggaCliHost3.extractActualBgpIntentNum( getIntentsResult )
+ bgpIntentsExpectedNum = 624
+ main.log.info( "bgpIntentsExpected num is:" )
+ main.log.info( bgpIntentsExpectedNum )
+ main.log.info( "bgpIntentsActual num is:" )
+ main.log.info( bgpIntentsActualNum)
+ utilities.assertEquals(
+ expect=True,
+ actual=eq( bgpIntentsExpectedNum, bgpIntentsActualNum ),
+ onpass="***PointToPointIntent Intent Num in SDN-IP are correct!***",
+ onfail="***PointToPointIntent Intent Num in SDN-IP are wrong!***" )
+ if ( eq( bgpIntentsExpectedNum, bgpIntentsActualNum ) ):
+ main.log.report(
+ "***PointToPointIntent Intents in SDN-IP are correct!***" )
+ else:
+ main.log.report(
+ "***PointToPointIntent Intents in SDN-IP are wrong!***" )
+
+ #============================= Ping Test ========================
+ # Wait until all MultiPointToSinglePoint intents are in system
+ time.sleep( 20 )
+ pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
+ pingTestResultsFile = \
+ "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-before-delete-routes-"\
+ + strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
+ pingTestResults = main.QuaggaCliHost.pingTest(
+ "1.168.30.100", pingTestScript, pingTestResultsFile )
+ main.log.info( pingTestResults )
+ time.sleep( 20 )
+
+ #============================= Deleting Routes ==================
+ main.step( "Check deleting routes installed" )
+ main.QuaggaCliHost3.deleteRoutes( prefixesHost3, 1 )
+ main.QuaggaCliHost4.deleteRoutes( prefixesHost4, 1 )
+ main.QuaggaCliHost5.deleteRoutes( prefixesHost5, 1 )
+
+ for i in range( 101, 201 ):
+ prefixesHostX = main.QuaggaCliHost.generatePrefixes( str( i ), 10 )
+ main.log.info( prefixesHostX )
+ QuaggaCliHostX = getattr( main, ( 'QuaggaCliHost' + str( i ) ) )
+ QuaggaCliHostX.deleteRoutes( prefixesHostX, 1 )
+
+ getRoutesResult = main.ONOScli.routes( jsonFormat=True )
+ allRoutesActual = \
+ main.QuaggaCliHost3.extractActualRoutesMaster( getRoutesResult )
+ main.log.info( "allRoutes_actual = " )
+ main.log.info( allRoutesActual )
+
+ utilities.assertEquals(
+ expect="[]", actual=str( allRoutesActual ),
+ onpass="***Route number in SDN-IP is 0, correct!***",
+ onfail="***Routes number in SDN-IP is not 0, wrong!***" )
+
+ if( eq( allRoutesStrExpected, allRoutesStrActual ) ):
+ main.log.report( "***Routes in SDN-IP after deleting correct!***" )
+ else:
+ main.log.report( "***Routes in SDN-IP after deleting wrong!***" )
+
+ main.step( "Check intents after deleting routes" )
+ getIntentsResult = main.ONOScli.intents( jsonFormat=True )
+ routeIntentsActualNum = \
+ main.QuaggaCliHost3.extractActualRouteIntentNum(
+ getIntentsResult )
+ main.log.info( "route Intents Actual Num is: " )
+ main.log.info( routeIntentsActualNum )
+ utilities.assertEquals(
+ expect=0, actual=routeIntentsActualNum,
+ onpass="***MultiPointToSinglePoint Intent Num in SDN-IP is 0, \
+ correct!***",
+ onfail="***MultiPointToSinglePoint Intent Num in SDN-IP is not 0, \
+ wrong!***" )
+
+ if( eq( 0, routeIntentsActualNum ) ):
+ main.log.report( "***MultiPointToSinglePoint Intents after \
+ deleting routes correct!***" )
+ else:
+ main.log.report( "***MultiPointToSinglePoint Intents after \
+ deleting routes wrong!***" )
+
+ time.sleep( 20 )
+ pingTestScript = "~/SDNIP/test-tools/CASE4-ping-as2host.sh"
+ pingTestResultsFile = \
+ "~/SDNIP/SdnIpIntentDemo/log/CASE4-ping-results-after-delete-routes-"\
+ + strftime( "%Y-%m-%d_%H:%M:%S", localtime() ) + ".txt"
+ pingTestResults = main.QuaggaCliHost.pingTest(
+ "1.168.30.100", pingTestScript, pingTestResultsFile )
+ main.log.info( pingTestResults )
+ time.sleep( 100 )
+
+ def CASE_4_origin( self, main ):
+ """
+ Test the SDN-IP functionality
+ allRoutesExpected: all expected routes for all BGP peers
+ routeIntentsExpected: all expected MultiPointToSinglePointIntent \
+ intents
+ bgpIntentsExpected: expected PointToPointIntent intents
+ allRoutesActual: all routes from ONOS LCI
+ routeIntentsActual: actual MultiPointToSinglePointIntent intents from \
+ ONOS CLI
+ bgpIntentsActual: actual PointToPointIntent intents from ONOS CLI
+ """
+ import time
+ import json
+ from operator import eq
+ from time import localtime, strftime
+
+ main.case("This case is to testing the functionality of SDN-IP with \
+ single ONOS instance" )
+ SDNIPJSONFILEPATH = \
+ "/home/admin/ONOS/tools/package/config/sdnip.json"
+ # all expected routes for all BGP peers
+ allRoutesExpected = []
+ main.step( "Start to generate routes for all BGP peers" )
+ main.log.info( "Generate prefixes for host3" )
+ prefixesHost3 = main.QuaggaCliHost3.generatePrefixes( 3, 10 )
+ main.log.info( prefixesHost3 )
+ # generate route with next hop
+ for prefix in prefixesHost3:
+ allRoutesExpected.append( prefix + "/" + "192.168.20.1" )
+ routeIntentsExpectedHost3 = \
+ main.QuaggaCliHost3.generateExpectedOnePeerRouteIntents(
+ prefixesHost3, "192.168.20.1", "00:00:00:00:02:02",
+ SDNIPJSONFILEPATH )
+
+ main.log.info( "Generate prefixes for host4" )
+ prefixesHost4 = main.QuaggaCliHost4.generatePrefixes( 4, 10 )
+ main.log.info( prefixesHost4 )
+ # generate route with next hop
+ for prefix in prefixesHost4:
+ allRoutesExpected.append( prefix + "/" + "192.168.30.1" )
+ routeIntentsExpectedHost4 = \
+ main.QuaggaCliHost4.generateExpectedOnePeerRouteIntents(
+ prefixesHost4, "192.168.30.1", "00:00:00:00:03:01",
+ SDNIPJSONFILEPATH )
+
+ main.log.info( "Generate prefixes for host5" )
+ prefixesHost5 = main.QuaggaCliHost5.generatePrefixes( 5, 10 )
+ main.log.info( prefixesHost5 )
+ for prefix in prefixesHost5:
+ allRoutesExpected.append( prefix + "/" + "192.168.60.2" )
+ routeIntentsExpectedHost5 = \
+ main.QuaggaCliHost5.generateExpectedOnePeerRouteIntents(
+ prefixesHost5, "192.168.60.1", "00:00:00:00:06:02",
+ SDNIPJSONFILEPATH )
+
+ routeIntentsExpected = routeIntentsExpectedHost3 + \
+ routeIntentsExpectedHost4 + routeIntentsExpectedHost5
+
+ main.step( "Get devices in the network" )
+ listResult = main.ONOScli.devices( jsonFormat=False )
+ main.log.info( listResult )
+ time.sleep( 10 )
+ if branchName == "onos-1.1":
+ main.log.info( "Installing sdn-ip feature" )
+ main.ONOScli.featureInstall( "onos-app-sdnip" )
+ else:
+ main.log.info( "Activate sdn-ip application" )
+ main.ONOScli.activateApp( "org.onosproject.sdnip" )
+ #main.log.info( "Installing sdn-ip feature" )
+ #main.ONOScli.featureInstall( "onos-app-sdnip" )
time.sleep( 10 )
main.step( "Login all BGP peers and add routes into peers" )
@@ -442,8 +702,12 @@
listResult = main.ONOScli.devices( jsonFormat=False )
main.log.info( listResult )
time.sleep( 10 )
- main.log.info( "Installing sdn-ip feature" )
- main.ONOScli.featureInstall( "onos-app-sdnip" )
+ if branchName == "onos-1.1":
+ main.log.info( "Installing sdn-ip feature" )
+ main.ONOScli.featureInstall( "onos-app-sdnip" )
+ else:
+ main.log.info( "Activate sdn-ip application" )
+ main.ONOScli.activateApp( "org.onosproject.sdnip" )
time.sleep( 10 )
main.step( "Login all BGP peers and add routes into peers" )