Merge "[ONOS-5280] Update FUNCintent and [ONOS-5357] Add more encap types"
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index ff3a56c..ebfd8a5 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -2216,21 +2216,27 @@
intentDictONOS = {}
for intent in intentsJson:
intentDictONOS[ intent[ 'id' ] ] = intent[ 'state' ]
+ returnValue = main.TRUE
if len( intentDict ) != len( intentDictONOS ):
- main.log.info( self.name + ": expected intent count does not match that in ONOS, " +
+ main.log.warn( self.name + ": expected intent count does not match that in ONOS, " +
str( len( intentDict ) ) + " expected and " +
str( len( intentDictONOS ) ) + " actual" )
- return main.FALSE
- returnValue = main.TRUE
+ returnValue = main.FALSE
for intentID in intentDict.keys():
if not intentID in intentDictONOS.keys():
main.log.debug( self.name + ": intent ID - " + intentID + " is not in ONOS" )
returnValue = main.FALSE
- elif intentDict[ intentID ] != intentDictONOS[ intentID ]:
- main.log.debug( self.name + ": intent ID - " + intentID +
- " expected state is " + intentDict[ intentID ] +
- " but actual state is " + intentDictONOS[ intentID ] )
- returnValue = main.FALSE
+ else:
+ if intentDict[ intentID ] != intentDictONOS[ intentID ]:
+ main.log.debug( self.name + ": intent ID - " + intentID +
+ " expected state is " + intentDict[ intentID ] +
+ " but actual state is " + intentDictONOS[ intentID ] )
+ returnValue = main.FALSE
+ intentDictONOS.pop( intentID )
+ if len( intentDictONOS ) > 0:
+ returnValue = main.FALSE
+ for intentID in intentDictONOS.keys():
+ main.log.debug( self.name + ": find extra intent in ONOS: intent ID " + intentID )
if returnValue == main.TRUE:
main.log.info( self.name + ": all intent IDs and states match that in ONOS" )
return returnValue
@@ -2654,14 +2660,20 @@
main.cleanup()
main.exit()
- def FlowAddedCount( self, deviceId ):
+ def flowAddedCount( self, deviceId, core=False ):
"""
Determine the number of flow rules for the given device id that are
in the added state
+ Params:
+ core: if True, only return the number of core flows added
"""
try:
- cmdStr = "flows any " + str( deviceId ) + " | " +\
- "grep 'state=ADDED' | wc -l"
+ if core:
+ cmdStr = "flows any " + str( deviceId ) + " | " +\
+ "grep 'state=ADDED' | grep org.onosproject.core | wc -l"
+ else:
+ cmdStr = "flows any " + str( deviceId ) + " | " +\
+ "grep 'state=ADDED' | wc -l"
handle = self.sendline( cmdStr )
assert "Command not found:" not in handle, handle
return handle
@@ -3369,7 +3381,7 @@
main.cleanup()
main.exit()
- def apps( self, jsonFormat=True ):
+ def apps( self, summary=False, active=False, jsonFormat=True ):
"""
Returns the output of the apps command for ONOS. This command lists
information about installed ONOS applications
@@ -3381,6 +3393,10 @@
# "features":"[onos-openflow]","state":"ACTIVE"}]
try:
cmdStr = "onos:apps"
+ if summary:
+ cmdStr += " -s"
+ if active:
+ cmdStr += " -a"
if jsonFormat:
cmdStr += " -j"
output = self.sendline( cmdStr )
@@ -4990,4 +5006,49 @@
return None
return respDic
+ def logSearch( self, searchTerm, mode='all' ):
+ """
+ Searches the latest ONOS log file for the given search term and
+ return a list that contains all the lines that have the search term.
+ Arguments:
+ searchTerm - A string to grep for in the ONOS log.
+ mode:
+ all: return all the strings that contain the search term
+ last: return the last string that contains the search term
+ first: return the first string that contains the search term
+ num: return the number that the searchTerm appears in the log
+ """
+ try:
+ assert type( searchTerm ) is str
+ cmd = "cat /opt/onos/log/karaf.log | grep \'" + searchTerm + "\'"
+ if mode == 'last':
+ cmd = cmd + " | tail -n 1"
+ if mode == 'first':
+ cmd = cmd + " | head -n 1"
+ if mode == 'num':
+ cmd = "cat /opt/onos/log/karaf.log | grep -c \'" + searchTerm + "\'"
+ num = self.sendline( cmd )
+ return num
+ before = self.sendline( cmd )
+ before = before.splitlines()
+ # make sure the returned list only contains the search term
+ returnLines = [line for line in before if searchTerm in line]
+ return returnLines
+ except AssertionError:
+ main.log.error( self.name + " searchTerm is not string type" )
+ return None
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+ except pexpect.TIMEOUT:
+ main.log.error( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+ except Exception:
+ main.log.exception( self.name + ": Uncaught exception!" )
+ main.cleanup()
+ main.exit()
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index 2f38e29..d46e99a 100755
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -388,6 +388,8 @@
# Prompt returned
break
main.log.debug( output )
+ # FIXME: This is a workaround for a bug in buck see ONOS-5320
+ self.buckPackage( )
return ret
except pexpect.TIMEOUT:
main.log.exception( self.name + ": TIMEOUT exception found" )
@@ -403,6 +405,53 @@
main.cleanup()
main.exit()
+ def buckPackage( self, timeout=180 ):
+ """
+ Package onos using buck. This will not build the source and this rule
+ should be automatically run when building onos.
+ """
+ try:
+ ret = main.TRUE
+ self.handle.sendline( "buck build package" )
+ self.handle.expect( "buck build package" )
+ output = ""
+ while True:
+ i = self.handle.expect( [ "This does not appear to be the root of a Buck project.",
+ "\n",
+ "BUILD FAILED",
+ "\$" ],
+ timeout=timeout )
+ output += str( self.handle.before + self.handle.after )
+ if i == 0:
+ main.log.error( "Wrong location" )
+ ret = main.FALSE
+ elif i == 1:
+ # end of a line, buck is still printing output
+ pass
+ elif i == 2:
+ # Build failed
+ main.log.error( "Build failed" )
+ ret = main.FALSE
+ elif i == 3:
+ # Prompt returned
+ break
+ main.log.debug( output )
+ return ret
+ except pexpect.TIMEOUT:
+ main.log.exception( self.name + ": TIMEOUT exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ return main.FALSE
+ except pexpect.EOF:
+ main.log.error( self.name + ": EOF exception found" )
+ main.log.error( self.name + ": " + self.handle.before )
+ main.cleanup()
+ main.exit()
+ except Exception:
+ main.log.exception( "Failed to package ONOS" )
+ main.cleanup()
+ main.exit()
+
+
def gitPull( self, comp1="", fastForward=True ):
"""
Assumes that "git pull" works without login
diff --git a/TestON/drivers/common/cli/ovsdbdriver.py b/TestON/drivers/common/cli/ovsdbdriver.py
index f3e0c7a..5921dc8 100644
--- a/TestON/drivers/common/cli/ovsdbdriver.py
+++ b/TestON/drivers/common/cli/ovsdbdriver.py
@@ -44,9 +44,9 @@
pwd=self.pwd)
if self.handle:
- return self.handle
main.log.onfo( "Connection successful to the ovsdb node " +
self.name )
+ return self.handle
else:
main.log.error( "Connection failed to the ovsdb node " +
self.name )
diff --git a/TestON/install.sh b/TestON/install.sh
index 0327ca7..fc434bc 100755
--- a/TestON/install.sh
+++ b/TestON/install.sh
@@ -217,6 +217,7 @@
if [ $# -eq 0 ]
then
+ init
default
elif [ $1 == "--help" ]
then
diff --git a/TestON/tests/CHOTestMonkey/CHOTestMonkey.params b/TestON/tests/CHOTestMonkey/CHOTestMonkey.params
index 6c9ef4b..85b2e0e 100644
--- a/TestON/tests/CHOTestMonkey/CHOTestMonkey.params
+++ b/TestON/tests/CHOTestMonkey/CHOTestMonkey.params
@@ -94,6 +94,8 @@
<CLIParamNum>0</CLIParamNum>
<rerunInterval>5</rerunInterval>
<maxRerunNum>5</maxRerunNum>
+ <coreFlowNum>4</coreFlowNum>
+ <coreFlowNum6>6</coreFlowNum6>
</FlowCheck>
<TrafficCheck>
diff --git a/TestON/tests/CHOTestMonkey/CHOTestMonkey.py b/TestON/tests/CHOTestMonkey/CHOTestMonkey.py
index 29a40aa..e6606d4 100644
--- a/TestON/tests/CHOTestMonkey/CHOTestMonkey.py
+++ b/TestON/tests/CHOTestMonkey/CHOTestMonkey.py
@@ -178,8 +178,8 @@
setIPv6CfgSleep = int( main.params[ 'TEST' ][ 'setIPv6CfgSleep' ] )
if main.enableIPv6:
time.sleep( setIPv6CfgSleep )
- cfgResult1 = main.controllers[ 0 ].CLI.setCfg( "org.onosproject.proxyarp.ProxyArp",
- "ipv6NeighborDiscovery",
+ cfgResult1 = main.controllers[ 0 ].CLI.setCfg( "org.onosproject.incubator.net.neighbour.impl.NeighbourResolutionManager",
+ "ndpEnabled",
"true" )
time.sleep( setIPv6CfgSleep )
cfgResult2 = main.controllers[ 0 ].CLI.setCfg( "org.onosproject.provider.host.impl.HostLocationProvider",
@@ -848,7 +848,7 @@
with main.eventScheduler.idleCondition:
while not main.eventScheduler.isIdle():
main.eventScheduler.idleCondition.wait()
- #time.sleep( sleepSec )
+ time.sleep( sleepSec )
utilities.assert_equals( expect=main.TRUE,
actual=main.caseResult,
onpass="Randomly generate events test passed",
diff --git a/TestON/tests/CHOTestMonkey/dependencies/events/CheckEvent.py b/TestON/tests/CHOTestMonkey/dependencies/events/CheckEvent.py
index aad520c..8c85ff8 100755
--- a/TestON/tests/CHOTestMonkey/dependencies/events/CheckEvent.py
+++ b/TestON/tests/CHOTestMonkey/dependencies/events/CheckEvent.py
@@ -46,9 +46,27 @@
def startCheckEvent( self, args=None ):
import json
checkResult = EventStates().PASS
+ if main.enableIPv6:
+ coreFlowNum = int( main.params[ 'EVENT' ][ 'FlowCheck' ][ 'coreFlowNum6' ] )
+ else:
+ coreFlowNum = int( main.params[ 'EVENT' ][ 'FlowCheck' ][ 'coreFlowNum' ] )
for controller in main.controllers:
if controller.isUp():
with controller.CLILock:
+ # Check core flow number
+ for device in main.devices:
+ if device.isRemoved():
+ continue
+ coreFlowNumOnos = controller.CLI.flowAddedCount( device.dpid, core=True )
+ if coreFlowNumOnos == None:
+ main.log.warn( "Flow Check - error when trying to get flow number of %s on ONOS%s" % ( device.dpid, controller.index ) )
+ checkResult = EventStates().FAIL
+ else:
+ coreFlowNumOnos = int( coreFlowNumOnos )
+ if coreFlowNumOnos != coreFlowNum:
+ main.log.warn( "Flow Check - core flow number of %s on ONOS%s is %s" % ( device.dpid, controller.index, coreFlowNumOnos ) )
+ checkResult = EventStates().FAIL
+ # Get flows for comparison
flows = controller.CLI.flows()
try:
flows = json.loads( flows )
diff --git a/TestON/tests/FUNC/FUNCipv6Intent/FUNCipv6Intent.py b/TestON/tests/FUNC/FUNCipv6Intent/FUNCipv6Intent.py
index 72fa1df..2f70420 100755
--- a/TestON/tests/FUNC/FUNCipv6Intent/FUNCipv6Intent.py
+++ b/TestON/tests/FUNC/FUNCipv6Intent/FUNCipv6Intent.py
@@ -6,7 +6,6 @@
self.default = ''
def CASE1( self, main ):
- import time
import imp
import re
@@ -127,7 +126,7 @@
- Install ONOS cluster
- Connect to cli
"""
-
+ import time
# main.scale[ 0 ] determines the current number of ONOS controller
main.numCtrls = int( main.scale[ 0 ] )
@@ -250,7 +249,7 @@
main.exit()
main.step( "setup the ipv6NeighbourDiscovery" )
- cfgResult1 = main.CLIs[0].setCfg( "org.onosproject.proxyarp.ProxyArp", "ipv6NeighborDiscovery", "true" )
+ cfgResult1 = main.CLIs[0].setCfg( "org.onosproject.incubator.net.neighbour.impl.NeighbourResolutionManager", "ndpEnabled", "true" )
cfgResult2 = main.CLIs[0].setCfg( "org.onosproject.provider.host.impl.HostLocationProvider", "ipv6NeighborDiscovery", "true" )
cfgResult = cfgResult1 and cfgResult2
utilities.assert_equals( expect=main.TRUE, actual=cfgResult,
@@ -338,7 +337,6 @@
"""
main.case( "Discover all hosts" )
- stepResult = main.TRUE
main.step( "Discover all hosts using pingall " )
stepResult = main.intentFunction.getHostsData( main )
utilities.assert_equals( expect=main.TRUE,
diff --git a/TestON/tests/FUNC/FUNCipv6Intent/dependencies/FUNCIpv6IntentFunction.py b/TestON/tests/FUNC/FUNCipv6Intent/dependencies/FUNCIpv6IntentFunction.py
index 0d71604..edca254 100755
--- a/TestON/tests/FUNC/FUNCipv6Intent/dependencies/FUNCIpv6IntentFunction.py
+++ b/TestON/tests/FUNC/FUNCipv6Intent/dependencies/FUNCIpv6IntentFunction.py
@@ -1795,7 +1795,7 @@
main.log.info( "Activating reactive forwarding app " )
activateResult = main.CLIs[ 0 ].activateApp( "org.onosproject.fwd" )
main.CLIs[ 0 ].setCfg( "org.onosproject.provider.host.impl.HostLocationProvider", "ipv6NeighborDiscovery", "true")
- main.CLIs[ 0 ].setCfg( "org.onosproject.proxyarp.ProxyArp", "ipv6NeighborDiscovery", "true")
+ main.CLIs[ 0 ].setCfg( "org.onosproject.incubator.net.neighbour.impl.NeighbourResolutionManager", "ndpEnabled", "true" )
main.CLIs[ 0 ].setCfg( "org.onosproject.fwd.ReactiveForwarding", "ipv6Forwarding", "true")
main.CLIs[ 0 ].setCfg( "org.onosproject.fwd.ReactiveForwarding", "matchIpv6Address", "true")
time.sleep( main.fwdSleep )
diff --git a/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.params b/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.params
index 0976c40..d166de3 100644
--- a/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.params
+++ b/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.params
@@ -36,6 +36,7 @@
<SLEEP>
<startup>15</startup>
<cfgGossip>2</cfgGossip>
+ <SetNetCfgSleep>5</SetNetCfgSleep>
</SLEEP>
<MININET>
diff --git a/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.py b/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.py
index 120d112..77e45ea 100644
--- a/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.py
+++ b/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.py
@@ -43,6 +43,7 @@
wrapperFile3 = main.params[ 'DEPENDENCY' ][ 'wrapper3' ]
main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
main.gossipTime = int( main.params[ 'SLEEP'][ 'cfgGossip' ] )
+ main.SetNetCfgSleep = int( main.params[ 'SLEEP' ][ 'SetNetCfgSleep' ] )
gitPull = main.params[ 'GIT' ][ 'pull' ]
main.cellData = {} # for creating cell file
main.hostsData = {}
@@ -402,6 +403,7 @@
Add some device configurations and then check they are distributed
to all nodes
"""
+ import time
main.case( "Add Network configurations to the cluster" )
main.caseExplanation = "Add Network Configurations for devices" +\
" not discovered yet. One device is allowed" +\
@@ -419,6 +421,8 @@
subjectKey="of:0000000000000001",
configKey="basic" )
s1Result = False
+ #Wait 5 secs after set up netCfg
+ time.sleep( main.SetNetCfgSleep )
if setS1Allow:
# Check what we set is what is in ONOS
getS1 = main.ONOSrest1.getNetCfg( subjectClass="devices",
@@ -427,6 +431,7 @@
onosCfg = pprint( getS1 )
sentCfg = pprint( s1Json )
if onosCfg == sentCfg:
+ main.log.info( "ONOS NetCfg match what was sent" )
s1Result = True
else:
main.log.error( "ONOS NetCfg doesn't match what was sent" )
@@ -449,6 +454,7 @@
subjectKey="of:0000000000000003",
configKey="basic" )
s3Result = False
+ time.sleep( main.SetNetCfgSleep )
if setS3Disallow:
# Check what we set is what is in ONOS
getS3 = main.ONOSrest1.getNetCfg( subjectClass="devices",
@@ -457,6 +463,7 @@
onosCfg = pprint( getS3 )
sentCfg = pprint( s3Json )
if onosCfg == sentCfg:
+ main.log.info("ONOS NetCfg match what was sent")
s3Result = True
else:
main.log.error( "ONOS NetCfg doesn't match what was sent" )
diff --git a/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.params b/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.params
index 0b5ad68..dba74d1 100644
--- a/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.params
+++ b/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.params
@@ -30,6 +30,10 @@
<delaytime>5</delaytime> #delaytime for ovsdb connection create and delete
</TIMER>
+ <SLEEP>
+ <startup>15</startup>
+ </SLEEP>
+
<HTTP>
<port>8181</port>
<path>/onos/vtn/</path>
diff --git a/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.py b/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.py
index 82274d0..91475c6 100644
--- a/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.py
+++ b/TestON/tests/FUNC/FUNCovsdbtest/FUNCovsdbtest.py
@@ -24,46 +24,46 @@
CASE1 is to compile ONOS and push it to the test machines
Startup sequence:
- cell <name>
- onos-verify-cell
+ Construct test variables
+ Safety check, kill all ONOS processes before setup
NOTE: temporary - onos-remove-raft-logs
- onos-uninstall
- start mininet
- git pull
- mvn clean install
- onos-package
- onos-install -f
- onos-wait-for-start
+ Create ONOS package
+ Install ONOS package
start cli sessions
start ovsdb
start vtn apps
"""
import os
- main.log.info( "ONOS Single node start " +
- "ovsdb test - initialization" )
+ import time
+ main.log.info( "ONOS Single node start ovsdb test - initialization" )
main.case( "Setting up test environment" )
main.caseExplanation = "Setup the test environment including " +\
"installing ONOS, start ONOS."
# load some variables from the params file
- PULLCODE = False
- if main.params[ 'GIT' ][ 'pull' ] == 'True':
- PULLCODE = True
+ main.step( "Constructing test variables" )
+ gitPull = main.params[ 'GIT' ][ 'pull' ]
gitBranch = main.params[ 'GIT' ][ 'branch' ]
cellName = main.params[ 'ENV' ][ 'cellName' ]
ipList = os.getenv( main.params[ 'CTRL' ][ 'ip1' ] )
- OVSDB1Ip = os.getenv( main.params[ 'OVSDB' ][ 'ip1' ] )
- OVSDB2Ip = os.getenv( main.params[ 'OVSDB' ][ 'ip2' ] )
+ main.startUpSleep = int( main.params['SLEEP']['startup'] )
+ cellAppString = main.params['ENV']['cellApps']
- main.step( "Create cell file" )
- cellAppString = main.params[ 'ENV' ][ 'cellApps' ]
- main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
- main.OVSDB1.ip_address,
- cellAppString, ipList )
+ if gitPull == 'True':
+ main.step( "Building ONOS in " + gitBranch + "branch" )
+ onosBuildResult = main.startUp.onosBuild( main, gitBranch )
+ stepResult = onosBuildResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully compiled latest ONOS",
+ onfail="Failed to compile latest ONOS")
+ else:
+ main.log.warn( "Did not pull new code so skipping mvn " +
+ "clean install" )
+ main.ONOSbench.getVersion( report=True )
- main.step( "Applying cell variable to environment" )
- cellResult = main.ONOSbench.setCell( cellName )
- verifyResult = main.ONOSbench.verifyCell()
+ main.log.info( "Safety check, killing all ONOS processes" +
+ " before initiating environment setup" )
main.log.info( "Removing raft logs" )
main.ONOSbench.onosRemoveRaftLogs()
@@ -80,9 +80,15 @@
except AttributeError:
break
- main.log.info( "Uninstalling ONOS" )
+ main.step( "Uninstalling ONOS package" )
+ onosUninstallResult = main.TRUE
for node in main.nodes:
- main.ONOSbench.onosUninstall( node.ip_address )
+ onosUninstallResult = onosUninstallResult and main.ONOSbench.onosUninstall( node.ip_address )
+ utilities.assert_equals( expect=main.TRUE,
+ actual=onosUninstallResult,
+ onpass="Successfully uninstalled ONOS package",
+ onfail="Failed to uninstall ONOS package" )
+ time.sleep( main.startUpSleep )
# Make sure ONOS process is not running
main.log.info( "Killing any ONOS processes" )
@@ -90,75 +96,83 @@
for node in main.nodes:
killed = main.ONOSbench.onosKill( node.ip_address )
killResults = killResults and killed
-
- cleanInstallResult = main.TRUE
- gitPullResult = main.TRUE
- main.step( "Git checkout and pull" + gitBranch )
- if PULLCODE:
- main.ONOSbench.gitCheckout( gitBranch )
- gitPullResult = main.ONOSbench.gitPull()
- # values of 1 or 3 are good
- utilities.assert_lesser( expect=0, actual=gitPullResult,
- onpass="Git pull successful",
- onfail="Git pull failed" )
-
- main.ONOSbench.getVersion( report=True )
-
- main.step( "Using mvn clean install" )
- cleanInstallResult = main.TRUE
- if PULLCODE and gitPullResult == main.TRUE:
- cleanInstallResult = main.ONOSbench.cleanInstall()
- else:
- main.log.warn( "Did not pull new code so skipping mvn" +
- "clean install" )
utilities.assert_equals( expect=main.TRUE,
- actual=cleanInstallResult,
- onpass="MCI successful",
- onfail="MCI failed" )
+ actual=onosUninstallResult,
+ onpass="Successfully kill all ONOS processes",
+ onfail="Failed to kill all ONOS processes" )
+
+ main.step( "Create cell file" )
+ main.ONOSbench.createCellFile( main.ONOSbench.ip_address, cellName,
+ main.OVSDB1.ip_address,
+ cellAppString, ipList )
+
+ main.step( "Apply cell to environment" )
+ cellResult = main.ONOSbench.setCell( cellName )
+ verifyResult = main.ONOSbench.verifyCell()
+ stepResult = cellResult and verifyResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully applied cell to environment",
+ onfail="Failed to apply cell to environment" )
+
main.step( "Creating ONOS package" )
packageResult = main.ONOSbench.buckBuild()
utilities.assert_equals( expect=main.TRUE,
- actual=packageResult,
- onpass="Successfully created ONOS package",
- onfail="Failed to create ONOS package" )
+ actual=packageResult,
+ onpass="Successfully created ONOS package",
+ onfail="Failed to create ONOS package" )
+ time.sleep( main.startUpSleep )
main.step( "Installing ONOS package" )
- onosInstallResult = main.ONOSbench.onosInstall(
- options="-f", node=main.nodes[0].ip_address )
+ onosInstallResult = main.ONOSbench.onosInstall( options="-f", node=main.nodes[0].ip_address )
utilities.assert_equals( expect=main.TRUE, actual=onosInstallResult,
- onpass="ONOS install successful",
- onfail="ONOS install failed" )
+ onpass="Successfully installed ONOS package",
+ onfail="Failed to install ONOS package" )
- main.step( "Checking if ONOS is up yet" )
- print main.nodes[0].ip_address
+ time.sleep( main.startUpSleep )
+ main.step("Starting ONOS service")
+ stopResult = main.TRUE
+ startResult = main.TRUE
+ onos1Isup = main.TRUE
for i in range( 2 ):
- onos1Isup = main.ONOSbench.isup( main.nodes[0].ip_address )
+ Isup = main.ONOSbench.isup( main.nodes[ 0 ].ip_address )
+ onos1Isup = onos1Isup and Isup
if onos1Isup:
- break
- utilities.assert_equals( expect=main.TRUE, actual=onos1Isup,
- onpass="ONOS startup successful",
- onfail="ONOS startup failed" )
+ main.log.report( "ONOS instance {0} is up and ready".format( i + 1 ) )
+ else:
+ main.log.report( "ONOS instance {0} may not be up, stop and ".format( i + 1 ) +
+ "start ONOS again" )
+ stopResult = stopResult and main.ONOSbench.onosStop( main.ONOSip[ i ] )
+ startResult = startResult and main.ONOSbench.onosStart( main.ONOSip[ i ] )
+ if not startResult or stopResult:
+ main.log.report( "ONOS instance {0} did not start correctly.".format( i + 1 ) )
+ stepResult = onos1Isup and stopResult and startResult
+ utilities.assert_equals( expect=main.TRUE, actual=stepResult,
+ onpass="ONOS service is ready on all nodes",
+ onfail="ONOS service did not start properly on all nodes" )
+
main.step( "Starting ONOS CLI sessions" )
- print main.nodes[0].ip_address
- cliResults = main.ONOScli1.startOnosCli( main.nodes[0].ip_address )
+ cliResults = main.ONOScli1.startOnosCli( main.nodes[ 0 ].ip_address )
utilities.assert_equals( expect=main.TRUE, actual=cliResults,
- onpass="ONOS cli startup successful",
- onfail="ONOS cli startup failed" )
+ onpass="Successfully start ONOS cli",
+ onfail="Failed to start ONOS cli" )
+
+ if cliResults == main.FALSE:
+ main.log.error( "Failed to start ONOS, stopping test" )
+ main.cleanup()
+ main.exit()
main.step( "App Ids check" )
appCheck = main.ONOScli1.appToIDCheck()
- if appCheck !=main.TRUE:
- main.log.warn( main.CLIs[0].apps() )
- main.log.warn( main.CLIs[0].appIDs() )
- utilities.assert_equals( expect=main.TRUE, actual=appCheck,
+ if appCheck != main.TRUE:
+ main.log.warn( main.CLIs[ 0 ].apps() )
+ main.log.warn( main.CLIs[ 0 ].appIDs() )
+
+ utilities.assert_equals( expect=main.TRUE, actual=appCheck,
onpass="App Ids seem to be correct",
onfail="Something is wrong with app Ids" )
- if cliResults == main.FALSE:
- main.log.error( "Failed to start ONOS,stopping test" )
- main.cleanup()
- main.exit()
main.step( "Install onos-ovsdb" )
installResults = main.ONOScli1.activateApp( "org.onosproject.ovsdb" )
@@ -177,9 +191,8 @@
"""
Test ovsdb connection and teardown
"""
- import os,sys
+ import os
import re
- import time
main.case( "Test ovsdb connection and teardown" )
main.caseExplanation = "Test ovsdb connection create and delete" +\
@@ -233,8 +246,7 @@
Test default br-int configuration and vxlan port
"""
import re
- import time
- import os,sys
+ import os
main.case( "Test default br-int configuration and vxlan port" )
main.caseExplanation = "onos create default br-int bridge and" +\
@@ -243,6 +255,8 @@
ctrlip = os.getenv( main.params[ 'CTRL' ][ 'ip1' ] )
ovsdbport = main.params[ 'CTRL' ][ 'ovsdbport' ]
delaytime = main.params[ 'TIMER' ][ 'delaytime' ]
+ OVSDB1Ip = os.getenv( main.params['OVSDB']['ip1'] )
+ OVSDB2Ip = os.getenv( main.params['OVSDB']['ip2'] )
main.step( "ovsdb node 1 set ovs manager to " + str( ctrlip ) )
assignResult = main.OVSDB1.setManager( ip=ctrlip, port=ovsdbport, delaytime=delaytime )
@@ -336,8 +350,7 @@
Test default openflow configuration
"""
import re
- import time
- import os,sys
+ import os
ctrlip = os.getenv( main.params[ 'CTRL' ][ 'ip1' ] )
ovsdbport = main.params[ 'CTRL' ][ 'ovsdbport' ]
@@ -441,8 +454,7 @@
Test default flows
"""
import re
- import time
- import os,sys
+ import os
ctrlip = os.getenv( main.params[ 'CTRL' ][ 'ip1' ] )
ovsdbport = main.params[ 'CTRL' ][ 'ovsdbport' ]
@@ -578,8 +590,7 @@
Test host go online and ping each other
"""
import re
- import time
- import os,sys
+ import os
ctrlip = os.getenv( main.params[ 'CTRL' ][ 'ip1' ] )
ovsdbport = main.params[ 'CTRL' ][ 'ovsdbport' ]
@@ -698,8 +709,7 @@
Clear ovs configuration and host configuration
"""
import re
- import time
- import os,sys
+ import os
ctrlip = os.getenv( main.params[ 'CTRL' ][ 'ip1' ] )
OVSDB1Ip = os.getenv( main.params[ 'OVSDB' ][ 'ip1' ] )
diff --git a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
index 862efb7..7fee993 100644
--- a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
+++ b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
@@ -182,7 +182,7 @@
# index = The number of the graph under plot name
job = "HAfullNetPartition"
plotName = "Plot-HA"
- index = "1"
+ index = "2"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
diff --git a/TestON/tests/HA/HAscaling/HAscaling.py b/TestON/tests/HA/HAscaling/HAscaling.py
index 9ab1fce..f3ccacf 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.py
+++ b/TestON/tests/HA/HAscaling/HAscaling.py
@@ -202,7 +202,7 @@
# index = The number of the graph under plot name
job = "HAscaling"
plotName = "Plot-HA"
- index = "0"
+ index = "1"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.py b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
index 84efd5d..24daf1d 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.py
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
@@ -196,7 +196,7 @@
# index = The number of the graph under plot name
job = "HAswapNodes"
plotName = "Plot-HA"
- index = "0"
+ index = "2"
graphs = '<ac:structured-macro ac:name="html">\n'
graphs += '<ac:plain-text-body><![CDATA[\n'
graphs += '<iframe src="https://onos-jenkins.onlab.us/job/' + job +\
diff --git a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
index 0abc6b7..9af6dae 100755
--- a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
+++ b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.params
@@ -18,10 +18,20 @@
<multiovs>multiovs.py</multiovs>
</DEPENDENCY>
+ <DATABASE>
+ <dbPath>/tmp/scaleTopoResultDb</dbPath>
+ </DATABASE>
+
<ENV>
<cellApps>drivers,openflow</cellApps>
</ENV>
+ <SearchTerm>
+ <start>New switch connection</start>
+ <end>Topology DefaultTopology</end>
+ <Disconnect>Switch disconnected callback</Disconnect>
+ </SearchTerm>
+
<GIT>
<pull>False</pull>
<branch>master</branch>
@@ -53,7 +63,7 @@
<TOPOLOGY>
<topology>torus</topology>
- <scale>20,25,30,35,40,45,50,55,60</scale>
+ <scale>5,10,20,30,35,40,50,55,60</scale>
<host>True</host>
</TOPOLOGY>
diff --git a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py
index 7babb49..2561e52 100644
--- a/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py
+++ b/TestON/tests/SCPF/SCPFscaleTopo/SCPFscaleTopo.py
@@ -20,11 +20,13 @@
- Install ONOS package
- Build ONOS package
"""
-
main.case( "Constructing test variables" )
main.step( "Constructing test variables" )
stepResult = main.FALSE
-
+ # The variable to decide if the data should be written into data base.
+ # 1 means Yes and -1 means No.
+ main.writeData = 1
+ main.searchTerm = main.params[ 'SearchTerm' ]
main.testOnDirectory = os.path.dirname( os.getcwd ( ) )
main.apps = main.params[ 'ENV' ][ 'cellApps' ]
gitBranch = main.params[ 'GIT' ][ 'branch' ]
@@ -64,6 +66,20 @@
for i in range(main.numCtrls):
main.CLIs.append( getattr( main, 'ONOScli%s' % (i+1) ) )
+ main.allinfo = {} # The dictionary to record all the data from karaf.log
+ for i in range( 2 ):
+ main.allinfo[ i ]={}
+ for w in range ( 3 ):
+ # Totaltime: the time from the new switchConnection to its end
+ # swConnection: the time from the first new switchConnection to the last new switchConnection
+ # disconnectRate: the rate that shows how many switch disconnect after connection
+ main.allinfo[ i ][ 'info' + str( w ) ]= { 'totalTime': 0, 'swConnection': 0,'disconnectRate': 0 }
+
+ main.dbFilePath = main.params[ 'DATABASE' ][ 'dbPath' ]
+ main.log.info( "Create Database file " + main.dbFilePath )
+ resultDB = open(main.dbFilePath, 'w+' )
+ resultDB.close()
+
main.startUp = imp.load_source( wrapperFile1,
main.dependencyPath +
wrapperFile1 +
@@ -253,11 +269,10 @@
" --switch ovsm --topo " + main.topoName + "," + main.currScale + "," + main.currScale
for i in range( main.numCtrls ):
mnCmd += " --controller remote,ip=" + main.ONOSip[ i ]
-
- stepResult = main.Mininet1.startNet(mnCmd=mnCmd)
+ stepResult = main.Mininet1.startNet( mnCmd=mnCmd )
utilities.assert_equals( expect=main.TRUE,
actual=stepResult,
- onpass=main.topoName +
+ onpass=main.topoName +
" topology started successfully",
onfail=main.topoName +
" topology failed to start" )
@@ -271,18 +286,26 @@
"""
import json
import time
+ # First capture
+ for i in range( 3 ):
+ # Calculate total time
+ main.allinfo[ 0 ][ 'info' + str( i )][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'end' ], 'last', index=i, funcMode='TD' )
+ # Calculate switch connection time
+ main.allinfo[ 0 ][ 'info' + str( i )][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'start' ], 'last', index=i, funcMode='TD' )
+ # Calculate the disconnecti rate
+ main.allinfo[ 0 ][ 'info' + str( i )][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'Disconnect' ], 'num', main.searchTerm[ 'start' ], 'num', index=i, funcMode='DR' )
+ main.log.debug( "The data is " + str( main.allinfo[ 0 ] ) )
- main.case( "Verifying topology: TORUS %sx%s" % (main.currScale, main.currScale) )
+ main.case( "Verifying topology: TORUS %sx%s" % ( main.currScale, main.currScale ) )
main.caseExplanation = "Pinging all hosts and comparing topology " +\
"elements between Mininet and ONOS"
- main.log.info( "Gathering topology information" )
+ main.log.info( "Gathering topology information")
time.sleep( main.MNSleep )
stepResult = main.TRUE
main.step( "Comparing MN topology to ONOS topology" )
-
- compareRetry=0
- while compareRetry <3:
+ compareRetry = 0
+ while compareRetry < 3:
#While loop for retry
devices = main.topo.getAllDevices( main )
ports = main.topo.getAllPorts( main )
@@ -429,8 +452,42 @@
'''
Report errors/warnings/exceptions
'''
+ # Compare the slowest Node through total time of each node
+ slowestNode = 0
+ slowestTotalTime = 0
+ # Second capture
+ for i in range( 3 ):
+ # Calculate total time
+ main.allinfo[ 1 ][ 'info' + str( i )][ 'totalTime' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'end' ], 'last', index=i, funcMode='TD' )
+ # Compare the total time
+ if main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ] > slowestTotalTime:
+ slowestTotalTime = main.allinfo[ 1 ][ 'info' + str( i ) ][ 'totalTime' ]
+ slowestNode = i
+ # Calculate switch connection time
+ main.allinfo[ 1 ][ 'info' + str( i )][ 'swConnection' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'start' ], 'first', main.searchTerm[ 'start' ], 'last', index=i, funcMode='TD' )
+ # Calculate the disconnecti rate
+ main.allinfo[ 1 ][ 'info' + str( i )][ 'disconnectRate' ] = main.scaleTopoFunction.getInfoFromLog( main, main.searchTerm[ 'Disconnect' ], 'num', main.searchTerm[ 'start' ],'num', index=i, funcMode='DR' )
+
+ if ( main.allinfo[ 0 ] != main.allinfo[ 1 ] ):
+ main.log.error( "The results of two capture are different!" )
+ main.log.debug( "The data is " + str( main.allinfo ) )
+ if main.writeData != -1:
+ main.log.info( "Write the date into database" )
+ # write the date into data base
+ with open( main.dbFilePath, "a" ) as dbFile:
+ temp = str( main.currScale )
+ temp += ",'baremetal1'"
+ # put result from second capture into data base
+ temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'totalTime' ] )
+ temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'swConnection' ] )
+ temp += "," + str( "%.2f" % main.allinfo[ 1 ][ 'info' + str( slowestNode )][ 'disconnectRate' ] )
+ temp += "\n"
+ dbFile.write( temp )
+ else:
+ main.log.error( "The data from log is wrong!" )
+ main.writeData = 1
main.case( "Checking logs for errors, warnings, and exceptions" )
- main.log.info("Error report: \n" )
+ main.log.info( "Error report: \n" )
main.ONOSbench.logReport( main.ONOSip[ 0 ],
[ "INFO",
"FOLLOWER",
diff --git a/TestON/tests/SCPF/SCPFscaleTopo/dependencies/scaleTopoFunction.py b/TestON/tests/SCPF/SCPFscaleTopo/dependencies/scaleTopoFunction.py
index 075f4f2..d05e2a7 100644
--- a/TestON/tests/SCPF/SCPFscaleTopo/dependencies/scaleTopoFunction.py
+++ b/TestON/tests/SCPF/SCPFscaleTopo/dependencies/scaleTopoFunction.py
@@ -9,6 +9,74 @@
def __init__( self ):
self.default = ''
+def getTimestampFromString( main, targetString ):
+ #Get time string from the target string
+ try:
+ assert type( targetString ) is str
+ timeString = targetString.split( ' | ' )
+ timeString = timeString[ 0 ]
+ from datetime import datetime
+ # convert time string to timestamp
+ t = datetime.strptime( timeString, "%Y-%m-%d %H:%M:%S,%f" )
+ import time
+ timestamp = time.mktime( t.timetuple() )
+ timestamp += int( t.microsecond / 1000 ) / 1000.0
+ return timestamp
+ except AssertionError:
+ main.log.error( "Got nothing firom log" )
+ return -1
+ except IndexError:
+ main.log.error( "Time string index error" )
+ return -1
+ except ValueError:
+ main.log.error( "Got wrong string from log" )
+ return -1
+
+def getInfoFromLog( main, term1, mode1, term2, mode2, index=0, funcMode='TD' ):
+ '''
+ Description:
+ Get needed informations of the search term from karaf.log
+ Includes onosclidriver functions
+ Function mode:
+ TD (time difference):
+ Get time difference between start and end
+ Term1: startTerm
+ Term2: endTerm
+ DR (disconnect rate):
+ Get switch disconnect rate
+ Term1: disconnectTerm
+ Term2: connectTerm
+
+ '''
+ try:
+ termInfo1 = main.CLIs[ index ].logSearch( term1, mode=mode1 )
+ termInfo2 = main.CLIs[ index ].logSearch( term2, mode=mode2 )
+ if funcMode == 'TD':
+ startTime = getTimestampFromString( main, termInfo1[0] )
+ endTime = getTimestampFromString ( main, termInfo2[0] )
+ if startTime == -1 or endTime == -1:
+ main.log.error( "Wrong Time!" )
+ main.writeData = -1
+ return -1
+ return endTime - startTime
+ if funcMode == 'DR':
+ #In this mode, termInfo1 means the total number of switch disconnection and
+ #termInfo2 means the total number of new switch connection
+ #termInfo2 - termInfo1 means the actual real number of switch connection.
+ disconnection = int( termInfo1 ) * 1.0
+ expectConnection = int( main.currScale ) ** 2
+ realConnection = int( termInfo2 ) - int( termInfo1 )
+ if expectConnection != realConnection:
+ main.log.error( "The number of real switch connection doesn't match the number of expected connection" )
+ main.writeData = -1
+ return -1
+ rate = disconnection / expectConnection
+ return rate
+ except IndexError:
+ main.log.error( "Catch the wrong information of search term" )
+ main.writeData = -1
+ return -1
+
def testTopology( main, topoFile='', args='', mnCmd='', timeout=300, clean=True ):
"""
Description:
diff --git a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
index fe3896e..c45fef8 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
+++ b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.params
@@ -66,6 +66,18 @@
<onosLogFile>/opt/onos/log/karaf*</onosLogFile>
<mci>off</mci>
+ <MaxWrong>20</MaxWrong>
+
+ <ResultRange>
+ <Min>0</Min>
+ <Max>500</Max>
+ </ResultRange>
+
+ <SearchTerm>
+ <up>DEVICE_ADDED</up>
+ <down>DEVICE_AVAILABILITY_CHANGED</down>
+ </SearchTerm>
+
<topoConfigFile>
single_topo_event_accumulator.cfg
</topoConfigFile>
@@ -81,6 +93,7 @@
<device>s3</device>
<tsharkResultPath>
<up>
+ <ALL>/tmp/Tshark_ALL</ALL>
<TCP>/tmp/Tshark_TCP</TCP>
<RQ>/tmp/Tshark_RQ</RQ> #role request
<RR>/tmp/Tshark_RR</RR> #role reply OF output
@@ -88,10 +101,10 @@
</up>
<down>
+ <ALL>/tmp/Tshark_ALL</ALL> #Fin_ack and Ack
<FA>/tmp/Tshark_FA</FA>#Fin_ack
<ACK>/tmp/Tshark_ACK</ACK>
</down>
-
</tsharkResultPath>
<singleSwThreshold>0,1000</singleSwThreshold>
@@ -103,6 +116,7 @@
<startup>5</startup>
<measure>5</measure>
<mininet>5</mininet>
+ <deleteSW>10</deleteSW>
</SLEEP>
<DATABASE>
diff --git a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
index 1ad1a59..ceacb73 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
+++ b/TestON/tests/SCPF/SCPFswitchLat/SCPFswitchLat.py
@@ -57,8 +57,14 @@
else:
main.log.warn( "Skipped pulling onos and Skipped building ONOS" )
-
- main.testOnDirectory = os.path.dirname(os.getcwd())
+ # The dictionary to record different type of wrongs
+ main.wrong = { 'totalWrong': 0, 'skipDown' : 0, 'TsharkValueIncorrect': 0,
+ 'TypeError' : 0, 'decodeJasonError': 0,
+ 'checkResultIncorrect': 0}
+ main.maxWrong = int( main.params['TEST'] ['MaxWrong'] )
+ main.resultRange = main.params['TEST']['ResultRange']
+ main.searchTerm = main.params['TEST']['SearchTerm']
+ main.testOnDirectory = os.path.dirname( os.getcwd() )
main.MN1Ip = main.params['MN']['ip1']
main.dependencyPath = main.testOnDirectory + \
main.params['DEPENDENCY']['path']
@@ -76,11 +82,11 @@
main.dbFileName = main.params['DATABASE']['dbName']
main.startUpSleep = int(main.params['SLEEP']['startup'])
main.measurementSleep = int( main.params['SLEEP']['measure'] )
+ main.deleteSwSleep = int( main.params['SLEEP']['deleteSW'] )
main.maxScale = int( main.params['max'] )
main.timeout = int( main.params['TIMEOUT']['timeout'] )
main.MNSleep = int( main.params['SLEEP']['mininet'])
main.device = main.params['TEST']['device']
-
main.log.info("Create Database file " + main.dbFileName)
resultsDB = open(main.dbFileName, "w+")
resultsDB.close()
@@ -202,6 +208,8 @@
"maxBatchMs 0")
main.CLIs[0].setCfg("org.onosproject.net.topology.impl.DefaultTopologyProvider",
"maxIdleMs 0")
+ for i in range(main.numCtrls):
+ main.CLIs[i].logSet( "DEBUG", "org.onosproject.metrics.topology")
time.sleep(1)
main.log.info("Copy topology file to Mininet")
@@ -251,11 +259,14 @@
"up", resultDict, True )
main.switchFunc.captureOfPack( main, main.device, main.ofPackage,
"down", resultDict, True )
+ main.CLIs[0].removeDevice( "of:0000000000000001" )
else:
main.switchFunc.captureOfPack( main, main.device, main.ofPackage,
"up", resultDict, False )
main.switchFunc.captureOfPack (main, main.device, main.ofPackage,
"down", resultDict, False )
+ main.CLIs[0].removeDevice( "of:0000000000000001" )
+
# Dictionary for result
maxDict = {}
maxDict['down'] = {}
diff --git a/TestON/tests/SCPF/SCPFswitchLat/dependencies/switchFunc.py b/TestON/tests/SCPF/SCPFswitchLat/dependencies/switchFunc.py
index 705e478..41a2fc4 100644
--- a/TestON/tests/SCPF/SCPFswitchLat/dependencies/switchFunc.py
+++ b/TestON/tests/SCPF/SCPFswitchLat/dependencies/switchFunc.py
@@ -6,40 +6,121 @@
'''
import time
import json
+
+def getTimestampFromLog( index, searchTerm ):
+ '''
+ Get timestamp value of the search term from log.
+ Args:
+ index: the index of cli
+ searchTerm: the key term of timestamp
+
+ '''
+ lines = main.CLIs[ index ].logSearch( searchTerm, mode='last' )
+ try:
+ assert lines != None
+ logString = lines[ len ( lines ) - 1 ]
+ #get the target value
+ line = logString.split( "time = " )
+ key = line[ 1 ].split( " " )
+ return int( key[ 0 ] )
+ except IndexError:
+ main.log.warn( "Index Error!" )
+ return 0
+ except AssertionError:
+ main.log.warn( "Search Term Not Found" )
+ return 0
+
def processPackage( package ):
'''
split package information to dictionary
Args:
package: Package String
- Returns:
-
'''
- pacakge = package.split(" ")
+ pacakge = package.split( " " )
dic = {}
for s in pacakge:
try:
- [key, value] = s.split("=")
- dic[key] = value
+ [ key, value ] = s.split( "=" )
+ dic[ key ] = value
except:
continue
return dic
-def findSeqBySeqAck( seq, packageList):
+def findSeqBySeqAck( seq, packageList ):
'''
Find specific Seq of package in packageList
Args:
seq: seq from last TCP package
packageList: find package in packageList
- Returns:
-
'''
for l in packageList:
- temp = processPackage(l)
- tA = temp['Ack']
- if int(seq) + 1 == int(tA):
- return temp['Seq']
+ temp = processPackage( l )
+ tA = temp[ 'Ack' ]
+ if int( seq ) + 1 == int( tA ):
+ return temp[ 'Seq' ]
+
+def arrangeTsharkFile( switchStatus, keyTerm ):
+ '''
+ Arrange different tshark messeage from overall file to different specific files
+ Args:
+ switchStatus: switch up or down
+ keyTerm: A dictionary that store the path name as value and the searchTerm as key
+
+ '''
+ with open( main.tsharkResultPath[ switchStatus ][ 'ALL' ], 'r' ) as resultFile:
+ resultText = resultFile.readlines()
+ resultFile.close()
+
+ for line in resultText:
+ for term in keyTerm:
+ if term in line:
+ path = '/tmp/Tshark_' + str( keyTerm[ term ] )
+ with open( path, 'a' ) as outputfile:
+ outputfile.write( line )
+ outputfile.close()
+
+def checkResult( result1, result2, result3 ):
+ '''
+ Check if the inputs meet the requirement
+ Returns:
+ 1 means the results are right, 0 means the results are wrong
+
+ '''
+ result = check( result1 ) + check( result2 ) + check( result3 )
+ if result < 3:
+ # if any result is wrong, increase the main wrong number
+ main.wrong[ 'checkResultIncorrect' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
+ return 0
+ return 1
+
+def check( result ):
+ '''
+ Check the single input.
+ Returns:
+ 1 means the input is good, 0 means the input is wrong
+
+ '''
+ if result < int( main.resultRange[ 'Min' ] ) or result > int( main.resultRange[ 'Max' ] ):
+ main.log.debug( str( result ) + " is not meet the requirement" )
+ return 0
+ return 1
+
+def checkTotalWrongNum():
+ '''
+ Check if the total wrong number is bigger than the max wrong number. If it is, then exit the
+ test.
+
+ '''
+ # if there are too many wrongs in this test, then exit
+ if main.wrong['totalWrong'] > main.maxWrong:
+ main.log.error( "The total wrong number exceeds %d, test terminated" % main.maxWrong )
+ main.cleanup()
+ main.exit()
+
def captureOfPack( main, deviceName, ofPack, switchStatus, resultDict, warmup ):
'''
@@ -52,203 +133,260 @@
resultDict: dictionary to contain result
warmup: warm up boolean
- Returns:
-
'''
- for d in ofPack[switchStatus]:
- main.log.info("Clean up Tshark")
- with open(main.tsharkResultPath[switchStatus][d], "w") as tshark:
- tshark.write("")
- main.log.info( "Starting tshark capture" )
- main.ONOSbench.tsharkGrep(ofPack[switchStatus][d], main.tsharkResultPath[switchStatus][d])
+ main.log.debug( "TOTAL WRONG: " + str( main.wrong ) )
+ for d in ofPack[ switchStatus ]:
+ main.log.info( "Clean up Tshark" )
+ with open( main.tsharkResultPath[ switchStatus ][ d ], "w" ) as tshark:
+ tshark.write( "" )
+ # use one tshark to grep everything
+ # Get the grep string
+ grepString = ''
+ keyTerm = {}
+ for d in ofPack[ switchStatus ]:
+ grepString = grepString + ofPack[ switchStatus ][ d ] + '|'
+ # get rid of regular experssion format
+ cleanTerm = ofPack[ switchStatus ][ d ].replace( '\\', '' )
+ keyTerm[ cleanTerm ] = d
+ # Delete the last '|'
+ grepString = grepString[:-1]
+ # open tshark
+ main.log.info( "starting tshark capture" )
+ main.ONOSbench.tsharkGrep( grepString, main.tsharkResultPath[ switchStatus ][ 'ALL' ], grepOptions='-E' )
if switchStatus == 'up':
# if up, assign switch to controller
- time.sleep(main.measurementSleep)
- main.log.info('Assigning {} to controller'.format(deviceName))
- main.Mininet1.assignSwController(sw=deviceName, ip=main.ONOSip[0])
- time.sleep(main.measurementSleep)
+ time.sleep( main.measurementSleep )
+ main.log.info( 'Assigning {} to controller'.format( deviceName ))
+ main.Mininet1.assignSwController( sw=deviceName, ip=main.ONOSip[0] )
+ time.sleep( main.measurementSleep )
if switchStatus == 'down':
# if down, remove switch from topology
- time.sleep(main.measurementSleep)
- main.step('Remove switch from controller')
- main.Mininet1.deleteSwController(deviceName)
- time.sleep(10)
+ time.sleep( main.measurementSleep )
+ main.step( 'Remove switch from controler' )
+ main.Mininet1.deleteSwController( deviceName )
+ time.sleep( main.deleteSwSleep )
main.log.info( "Stopping all Tshark processes" )
main.ONOSbench.tsharkStop()
tempResultDict = {}
+ arrangeTsharkFile( switchStatus, keyTerm )
+
if switchStatus == 'up':
- for d in main.tsharkResultPath['up']:
- with open(main.tsharkResultPath[switchStatus][d], "r") as resultFile:
+ for d in main.tsharkResultPath[ 'up' ]:
+ with open( main.tsharkResultPath[ switchStatus ][ d ], "r" ) as resultFile:
# grep tshark result timestamp
resultText = resultFile.readlines()
if d == "TCP":
# if TCP package, we should use the latest one package
- resultText = resultText[len(resultText) - 1]
+ resultText = resultText[ len( resultText ) - 1 ]
else:
- resultText = resultText[0]
- main.log.info("Capture result:" + resultText)
+ resultText = resultText[ 0 ]
+ main.log.info( "Capture result:" + resultText )
resultText = resultText.strip()
resultText = resultText.split( " " )
if len(resultText) > 1:
- tempResultDict[d]= int( ( float(resultText[1]) * 1000 ) )
+ tempResultDict[d]= int( ( float( resultText[ 1 ] ) * 1000 ) )
resultFile.close()
elif switchStatus == 'down':
# if state is down, we should capture Fin/Ack and ACK package
# Use seq number in FIN/ACK package to located ACK package
- with open(main.tsharkResultPath['down']['FA']) as resultFile:
+ with open( main.tsharkResultPath[ 'down' ][ 'FA' ], 'r' ) as resultFile:
resultText = resultFile.readlines()
- FinAckText = resultText.pop(0)
+ FinAckText = resultText.pop( 0 )
resultFile.close()
- FinAckSeq = processPackage(FinAckText)['Seq']
- FinAckOFseq = findSeqBySeqAck(FinAckSeq, resultText)
-
- with open(main.tsharkResultPath['down']['ACK']) as resultFile:
+ FinAckSeq = processPackage( FinAckText )[ 'Seq' ]
+ FinAckOFseq = findSeqBySeqAck( FinAckSeq, resultText )
+ with open( main.tsharkResultPath[ 'down' ][ 'ACK' ], "r" ) as resultFile:
ACKlines = resultFile.readlines()
resultFile.close()
-
AckPackage = ""
for l in ACKlines:
- temp = processPackage(l)
- if temp['Seq'] == findSeqBySeqAck(FinAckOFseq, ACKlines):
+ temp = processPackage( l )
+ finSeq = findSeqBySeqAck( FinAckOFseq, ACKlines )
+ if temp[ 'Seq' ] == finSeq:
AckPackage = l
- if len(AckPackage) > 0:
+ if len( AckPackage ) > 0:
FinAckText = FinAckText.strip()
- FinAckText = FinAckText.split(" ")
+ FinAckText = FinAckText.split( " " )
AckPackage = AckPackage.strip()
- AckPackage = AckPackage.split(" ")
- tempResultDict['ACK'] = int(float(AckPackage[1]) * 1000)
- tempResultDict['FA'] = int(float(FinAckText[1]) * 1000)
+ AckPackage = AckPackage.split( " " )
+ tempResultDict[ 'ACK' ] = int( float( AckPackage[ 1 ] ) * 1000 )
+ tempResultDict[ 'FA' ] = int( float( FinAckText[ 1 ] ) * 1000 )
else:
+ main.wrong[ 'skipDown' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
return
+
# calculate latency
if switchStatus == "up":
# up Latency
- for d in resultDict[switchStatus]:
+ for d in resultDict[ switchStatus ]:
T_Ftemp = 0
F_Rtemp = 0
RQ_RRtemp = 0
try:
- T_Ftemp = tempResultDict['Feature'] - tempResultDict['TCP']
- F_Rtemp = tempResultDict['RQ'] - tempResultDict['Feature']
- RQ_RRtemp = tempResultDict['RR'] - tempResultDict['RQ']
+ T_Ftemp = tempResultDict[ 'Feature' ] - tempResultDict[ 'TCP' ]
+ F_Rtemp = tempResultDict[ 'RQ' ] - tempResultDict[ 'Feature' ]
+ RQ_RRtemp = tempResultDict[ 'RR' ] - tempResultDict[ 'RQ' ]
except KeyError:
- main.log.warn("Tshark Result was incorrect!")
- main.log.warn(tempResultDict)
+ main.log.warn( "Tshark Result was incorrect!" )
+ main.log.warn( tempResultDict )
+ main.wrong[ 'TsharkValueIncorrect' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
return
if not warmup:
- resultDict[switchStatus][d][ 'T_F' ].append( T_Ftemp )
- resultDict[switchStatus][d][ 'F_R' ].append( F_Rtemp )
- resultDict[switchStatus][d][ 'RQ_RR' ].append( RQ_RRtemp )
+ resultDict[ switchStatus ][ d ][ 'T_F' ].append( T_Ftemp )
+ resultDict[ switchStatus ][ d ][ 'F_R' ].append( F_Rtemp )
+ resultDict[ switchStatus ][ d ][ 'RQ_RR' ].append( RQ_RRtemp )
- main.log.info("{} TCP to Feature: {}".format(d, str( T_Ftemp ) ) )
- main.log.info("{} Feature to Role Request: {}".format(d, str(F_Rtemp)))
- main.log.info("{} Role Request to Role Reply: {}".format(d, str(RQ_RRtemp)))
+ main.log.info( "{} TCP to Feature: {}".format( d, str( T_Ftemp ) ) )
+ main.log.info( "{} Feature to Role Request: {}".format( d, str( F_Rtemp ) ) )
+ main.log.info( "{} Role Request to Role Reply: {}".format( d, str( RQ_RRtemp ) ) )
- for i in range(1, main.numCtrls + 1):
+ for i in range( 1, main.numCtrls + 1 ):
RR_Dtemp = 0
D_Gtemp = 0
E_Etemp = 0
- main.log.info("================================================")
+ main.log.info( "================================================" )
# get onos metrics timestamps
try:
- response = json.loads(main.CLIs[i - 1].topologyEventsMetrics())
- DeviceTime = int( response.get("topologyDeviceEventTimestamp").get("value") )
- main.log.info("ONOS{} device Event timestamp: {}".format(i, "%.2f" % DeviceTime))
- GraphTime = int( response.get("topologyGraphEventTimestamp").get("value") )
- main.log.info("ONOS{} Graph Event timestamp: {}".format(i, GraphTime))
+ response = json.loads( main.CLIs[i - 1].topologyEventsMetrics() )
+ DeviceTime = getTimestampFromLog( i - 1, searchTerm=main.searchTerm[switchStatus] )
+ main.log.info( "ONOS{} device Event timestamp: {}".format( i, "%.2f" % DeviceTime ) )
+ GraphTime = int( response.get( "topologyGraphEventTimestamp" ).get( "value" ) )
+ main.log.info( "ONOS{} Graph Event timestamp: {}".format( i, GraphTime ) )
except TypeError:
- main.log.warn("TypeError")
+ main.log.warn( "TypeError" )
+ main.wrong[ 'TypeError' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
break
except ValueError:
- main.log.warn("Error to decode Json object!")
+ main.log.warn( "Error to decode Json object!" )
+ main.wrong[ 'decodeJasonError' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
break
- try:
- #FIXME: the Device Event (PORT_ADD) we got from metrics app is not the one generated Graph Event
- if DeviceTime > GraphTime:
- # This fixes the negative latency values. However we are not using the right Device Event
- # timestamp. This should be fixed later.
- DeviceTime = GraphTime
- RR_Dtemp = DeviceTime - tempResultDict['RR']
- D_Gtemp = GraphTime - DeviceTime
- E_Etemp = GraphTime - tempResultDict['TCP']
- main.log.info("Role reply to Device:{}".format(RR_Dtemp))
- main.log.info("Device to Graph:{}".format(D_Gtemp))
- main.log.info("End to End:{}".format(E_Etemp))
- main.log.info("================================================")
- except KeyError:
- main.log.warn("Tshark Result was incorrect!")
- main.log.warn(tempResultDict)
- return
- except TypeError:
- main.log.warn("TypeError")
- break
- except ValueError:
- main.log.warn("Error to decode Json object!")
- break
- if not warmup:
- resultDict[switchStatus]['node' + str(i)][ 'RR_D' ].append( RR_Dtemp )
- resultDict[switchStatus]['node' + str(i)][ 'D_G' ].append( D_Gtemp )
- resultDict[switchStatus]['node' + str(i)][ 'E_E' ].append( E_Etemp )
-
- main.log.info( "Node {} Role Reply to Device: {}".format(str(i), str(RR_Dtemp) ) )
- main.log.info( "Node {} Device to Graph: {}".format(str(i), str(D_Gtemp) ) )
- main.log.info( "Node {} End to End: {}".format(str(i), str(E_Etemp) ) )
+ if DeviceTime != 0:
+ try:
+ RR_Dtemp = DeviceTime - tempResultDict[ 'RR' ]
+ D_Gtemp = GraphTime - DeviceTime
+ E_Etemp = GraphTime - tempResultDict[ 'TCP' ]
+ check = checkResult( RR_Dtemp, D_Gtemp, E_Etemp )
+ if check == 1:
+ main.log.info( "Role reply to Device:{}".format( RR_Dtemp ) )
+ main.log.info( "Device to Graph:{}".format( D_Gtemp ) )
+ main.log.info( "End to End:{}".format( E_Etemp ) )
+ main.log.info( "================================================" )
+ except KeyError:
+ main.log.warn( "Tshark Result was incorrect!" )
+ main.log.warn( tempResultDict )
+ main.wrong[ 'TsharkValueIncorrect' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
+ return
+ except TypeError:
+ main.log.warn( "TypeError" )
+ main.wrong[ 'TypeError' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
+ break
+ except ValueError:
+ main.log.warn( "Error to decode Json object!" )
+ main.wrong[ 'decodeJasonError' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
+ break
+ if not warmup and check == 1:
+ resultDict[ switchStatus ][ 'node' + str( i )][ 'RR_D' ].append( RR_Dtemp )
+ resultDict[ switchStatus ][ 'node' + str( i )][ 'D_G' ].append( D_Gtemp )
+ resultDict[ switchStatus ][ 'node' + str( i )][ 'E_E' ].append( E_Etemp )
+ else:
+ main.wrong['checkResultIncorrect'] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
+ main.log.debug("Skip this iteration due to the None Devicetime")
if switchStatus == "down":
# down Latency
- for d in resultDict[switchStatus]:
+ for d in resultDict[ switchStatus ]:
FA_Atemp = 0
try:
- FA_Atemp = tempResultDict['ACK'] - tempResultDict['FA']
+ FA_Atemp = tempResultDict[ 'ACK' ] - tempResultDict[ 'FA' ]
except KeyError:
- main.log.warn("Tshark Result was incorrect!")
- main.log.warn(tempResultDict)
+ main.log.warn( "Tshark Result was incorrect!" )
+ main.log.warn( tempResultDict )
+ main.wrong[ 'TsharkValueIncorrect' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
return
-
if not warmup:
- resultDict[switchStatus][d][ 'FA_A' ].append( FA_Atemp )
- main.log.info( "{} FIN/ACK TO ACK {}:".format(d , FA_Atemp) )
- for i in range(1, main.numCtrls + 1):
+ resultDict[ switchStatus ][ d ][ 'FA_A' ].append( FA_Atemp )
+ main.log.info( "{} FIN/ACK TO ACK {}:".format( d, FA_Atemp ) )
+ for i in range( 1, main.numCtrls + 1 ):
A_Dtemp = 0
D_Gtemp = 0
E_Etemp = 0
-
- main.log.info("================================================")
+ main.log.info( "================================================" )
# get onos metrics timestamps
try:
- response = json.loads(main.CLIs[i - 1].topologyEventsMetrics())
- DeviceTime = int( response.get("topologyDeviceEventTimestamp").get("value") )
- main.log.info("ONOS{} device Event timestamp: {}".format(i, DeviceTime))
- GraphTime = int( response.get("topologyGraphEventTimestamp").get("value") )
- main.log.info("ONOS{} Graph Event timestamp: {}".format(i, GraphTime))
+ response = json.loads( main.CLIs[ i - 1 ].topologyEventsMetrics() )
+ DeviceTime = getTimestampFromLog( i - 1, searchTerm=main.searchTerm[switchStatus] )
+ main.log.info( "ONOS{} device Event timestamp: {}".format( i, DeviceTime ) )
+ GraphTime = int( response.get( "topologyGraphEventTimestamp" ).get( "value" ) )
+ main.log.info( "ONOS{} Graph Event timestamp: {}".format( i, GraphTime ) )
except TypeError:
- main.log.warn("TypeError")
+ main.log.warn( "TypeError" )
+ main.wrong[ 'TypeError' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
break
except ValueError:
- main.log.warn("Error to decode Json object!")
+ main.log.warn( "Error to decode Json object!" )
+ main.wrong[ 'decodeJasonError' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
break
- main.log.info("================================================")
- try:
- A_Dtemp = DeviceTime - tempResultDict['ACK']
- D_Gtemp = GraphTime - DeviceTime
- E_Etemp = GraphTime - tempResultDict['FA']
- main.log.info("ACK to device: {}".format(A_Dtemp))
- main.log.info("Device ot Graph: {}".format(D_Gtemp))
- main.log.info("End to End: {}".format(E_Etemp))
- main.log.info("================================================")
- except KeyError:
- main.log.warn("Tshark Result was incorrect!")
- main.log.warn(tempResultDict)
- return
- except TypeError:
- main.log.warn("TypeError")
- break
- except ValueError:
- main.log.warn("Error to decode Json object!")
- break
- if not warmup:
- resultDict[switchStatus]['node' + str(i)][ 'A_D' ].append( A_Dtemp )
- resultDict[switchStatus]['node' + str(i)][ 'D_G' ].append( D_Gtemp )
- resultDict[switchStatus]['node' + str(i)][ 'E_E' ].append( E_Etemp )
- main.CLIs[0].removeDevice( "of:0000000000000001" )
+ if DeviceTime != 0:
+ main.log.info( "================================================" )
+ try:
+ A_Dtemp = DeviceTime - tempResultDict[ 'ACK' ]
+ D_Gtemp = GraphTime - DeviceTime
+ E_Etemp = GraphTime - tempResultDict[ 'FA' ]
+ check = checkResult( A_Dtemp, D_Gtemp, E_Etemp )
+ if check == 1:
+ main.log.info( "ACK to device: {}".format( A_Dtemp ) )
+ main.log.info( "Device ot Graph: {}".format( D_Gtemp ) )
+ main.log.info( "End to End: {}".format( E_Etemp ) )
+ main.log.info( "================================================" )
+ except KeyError:
+ main.log.warn( "Tshark Result was incorrect!" )
+ main.log.warn( tempResultDict )
+ main.wrong[ 'TsharkValueIncorrect' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
+ return
+ except TypeError:
+ main.log.warn( "TypeError" )
+ main.wrong[ 'TypeError' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
+ break
+ except ValueError:
+ main.log.warn( "Error to decode Json object!" )
+ main.wrong[ 'decodeJasonError' ] += 1
+ main.wrong[ 'totalWrong' ] += 1
+ checkTotalWrongNum()
+ break
+ if not warmup and check == 1:
+ resultDict[ switchStatus ][ 'node' + str( i ) ][ 'A_D' ].append( A_Dtemp )
+ resultDict[ switchStatus ][ 'node' + str( i ) ][ 'D_G' ].append( D_Gtemp )
+ resultDict[ switchStatus ][ 'node' + str( i ) ][ 'E_E' ].append( E_Etemp )
+
+ else:
+ main.wrong['checkResultIncorrect'] += 1
+ main.wrong['totalWrong'] += 1
+ checkTotalWrongNum()
+ main.log.debug("Skip this iteration due to the None Devicetime")
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/README.md b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/README.md
new file mode 100644
index 0000000..f057e2c
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/README.md
@@ -0,0 +1,17 @@
+These tests are meant to test the high availability of ONOS and
+SR application.
+
+It consists of:
+1) Configure and install ONOS;
+2) Pingall between hosts;
+3) Kill one ONOS instance;
+4) Kill one spine;
+5) Repeat this test a number of time;
+
+Requirements:
+1) An updated version of the CPQD switch has to be running to make sure it supports group chaining.
+
+The test is executed using the netcfg subsystem:
+1) APPS=openflow-base,netcfghostprovider,netcfglinksprovider
+
+The topologies are 2x2 Leaf-Spine and 4x4 Leaf-Spine.
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params
new file mode 100644
index 0000000..5f990fa
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.params
@@ -0,0 +1,53 @@
+<PARAMS>
+
+ <testcases>1,2,3,4,5,6,7,8</testcases>
+
+ <SCALE>
+ <size>3</size>
+ <max>3</max>
+ </SCALE>
+
+ <DEPENDENCY>
+ <wrapper1>startUp</wrapper1>
+ <topology>cord_fabric.py</topology>
+ </DEPENDENCY>
+
+ <ENV>
+ <cellName>productionCell</cellName>
+ <cellApps>drivers,segmentrouting</cellApps>
+ <diffApps>openflow-base,netcfghostprovider,netcfglinksprovider</diffApps>
+ <cellUser>sdn</cellUser>
+ </ENV>
+
+ <GIT>
+ <pull>False</pull>
+ <branch>master</branch>
+ </GIT>
+
+ <CTRL>
+ <port>6653</port>
+ </CTRL>
+
+ <timers>
+ <LinkDiscovery>12</LinkDiscovery>
+ <SwitchDiscovery>12</SwitchDiscovery>
+ </timers>
+
+ <switches>
+ <spine1> spine101 </spine1>
+ <spine2> spine102 </spine2>
+ <spine3> spine103 </spine3>
+ <spine4> spine104 </spine4>
+ <spinedpid1> 000000000101 </spinedpid1>
+ <spinedpid2> 000000000102 </spinedpid2>
+ <spinedpid3> 000000000103 </spinedpid3>
+ <spinedpid4> 000000000104 </spinedpid4>
+ </switches>
+
+ <failures>3</failures>
+
+ <SLEEP>
+ <startup>10</startup>
+ </SLEEP>
+
+</PARAMS>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.py b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.py
new file mode 100644
index 0000000..77f7d71
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.py
@@ -0,0 +1,318 @@
+# In this test we perform several failures and then test for connectivity
+# CASE1: 2x2 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures
+# CASE2: 2x2 topo + 3 ONOS + | ONOS (random instance) failure + IP connectivity test | x failures
+# CASE3: 4x4 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures
+# CASE4: 4x4 topo + 3 ONOS + | ONOS (random instance) failure + IP connectivity test | x failures
+# CASE5: 2x2 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures
+# CASE6: 2x2 topo + 3 ONOS + | ONOS (random instance) failure + Spine (random switch) failure + IP connectivity test | x failures
+# CASE7: 4x4 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures
+# CASE8: 4x4 topo + 3 ONOS + | ONOS (random instance) failure + Spine (random switch) failure + IP connectivity test | x failures
+
+
+
+class SRHighAvailability:
+
+ def __init__( self ):
+ self.default = ''
+
+ def CASE1( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 2x2 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause sequential ONOS failure
+ 5) Pingall
+ 6) Repeat 3), 4), 5) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS failures with 2x2 Leaf-spine "
+ main.case( description )
+ run.config(main, '2x2', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ toKill = i % main.numCtrls
+ run.killOnos( main, [ toKill ], '4', '8', '2' )
+ run.pingAll( main, 'CASE1_Failure%d' % (i+1) )
+ run.recoverOnos( main, [ toKill ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE1_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE2( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 2x2 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause random ONOS failure
+ 5) Pingall
+ 6) Repeat 3), 4), 5) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import random
+ from random import randint
+ from datetime import datetime
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS random failures with 2x2 Leaf-spine "
+ main.case( description )
+ run.config(main, '2x2', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main )
+ random.seed(datetime.now())
+ for i in range(0, main.failures):
+ toKill = randint(0, (main.numCtrls-1))
+ run.killOnos( main, [ toKill ], '4', '8', '2' )
+ run.pingAll( main, 'CASE2_Failure%d' % (i+1) )
+ run.recoverOnos( main, [ toKill ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE2_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE3( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 4x4 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause sequential ONOS failure
+ 5) Pingall
+ 6) Repeat 3), 4), 5) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS failures with 4x4 Leaf-spine "
+ main.case( description )
+ run.config(main, '4x4', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ toKill = i % main.numCtrls
+ run.killOnos( main, [ toKill ], '8', '32', '2' )
+ run.pingAll( main, 'CASE3_Failure%d' % (i+1) )
+ run.recoverOnos( main, [ toKill ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE3_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE4( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 4x4 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause random ONOS failure
+ 5) Pingall
+ 6) Repeat 3), 4), 5) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import random
+ from random import randint
+ from datetime import datetime
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS random failures with 4x4 Leaf-spine "
+ main.case( description )
+ run.config(main, '4x4', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main )
+ random.seed(datetime.now())
+ for i in range(0, main.failures):
+ toKill = randint(0, (main.numCtrls-1))
+ run.killOnos( main, [ toKill ], '8', '32', '2' )
+ run.pingAll( main, 'CASE4_Failure%d' % (i+1) )
+ run.recoverOnos( main, [ toKill ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE4_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE5( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 2x2 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause sequential ONOS failure
+ 5) Pingall
+ 6) Cause sequential Spine failure
+ 7) Pingall
+ 8) Repeat 3), 4), 5), 6), 7), 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import time
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS failures and Switch failures with 2x2 Leaf-spine "
+ main.case( description )
+ run.config(main, '2x2', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ onosToKill = i % main.numCtrls
+ switchToKill = i % len(main.spines)
+ run.killOnos( main, [ onosToKill ], '4', '8', '2' )
+ run.pingAll( main, 'CASE5_ONOS_Failure%d' % (i+1) )
+ run.killSwitch( main, main.spines[switchToKill]['name'], switches='3', links='4' )
+ time.sleep( main.switchSleep )
+ run.pingAll( main, "CASE5_SWITCH_Failure%d" % (i+1) )
+ run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='4', links='8' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, "CASE5_SWITCH_Recovery%d" % (i+1) )
+ run.recoverOnos( main, [ onosToKill ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE5_ONOS_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE6( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 2x2 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause random ONOS failure
+ 5) Pingall
+ 6) Cause random Spine failure
+ 7) Pingall
+ 8) Repeat 3), 4), 5), 6), 7) 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import time
+ import random
+ from random import randint
+ from datetime import datetime
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS random failures and Switch random failures with 2x2 Leaf-spine "
+ main.case( description )
+ run.config(main, '2x2', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py' )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ onosToKill = randint(0, (main.numCtrls-1))
+ switchToKill = randint(0, 1)
+ run.killOnos( main, [ onosToKill ], '4', '8', '2' )
+ run.pingAll( main, 'CASE6_ONOS_Failure%d' % (i+1) )
+ run.killSwitch( main, main.spines[switchToKill]['name'], switches='3', links='4' )
+ time.sleep( main.switchSleep )
+ run.pingAll( main, "CASE6_SWITCH_Failure%d" % (i+1) )
+ run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='4', links='8' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, "CASE6_SWITCH_Recovery%d" % (i+1) )
+ run.recoverOnos( main, [ onosToKill ], '4', '8', '3' )
+ run.checkFlows( main, minFlowCount=116 )
+ run.pingAll( main, 'CASE6_ONOS_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE7( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 4x4 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause sequential ONOS failure
+ 5) Pingall
+ 6) Cause sequential Spine failure
+ 7) Pingall
+ 8) Repeat 3), 4), 5), 6), 7), 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import time
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS failures and Switch failures with 4x4 Leaf-spine "
+ main.case( description )
+ run.config(main, '4x4', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ onosToKill = i % main.numCtrls
+ switchToKill = i % len(main.spines)
+ run.killOnos( main, [ onosToKill ], '8', '32', '2' )
+ run.pingAll( main, 'CASE7_ONOS_Failure%d' % (i+1) )
+ run.killSwitch( main, main.spines[switchToKill]['name'], switches='7', links='24' )
+ time.sleep( main.switchSleep )
+ run.pingAll( main, "CASE7_SWITCH_Failure%d" % (i+1) )
+ run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='8', links='32' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, "CASE7_SWITCH_Recovery%d" % (i+1) )
+ run.recoverOnos( main, [ onosToKill ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE7_ONOS_Recovery%d' % (i+1) )
+ run.cleanup( main )
+
+ def CASE8( self, main ):
+ """
+ 1) Sets up 3-nodes Onos-cluster
+ 2) Start 4x4 Leaf-Spine topology
+ 3) Pingall
+ 4) Cause random ONOS failure
+ 5) Pingall
+ 6) Cause random Spine failure
+ 7) Pingall
+ 8) Repeat 3), 4), 5), 6), 7), 'failures' times
+ """
+ from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
+ Testcaselib as run
+ import time
+ import random
+ from random import randint
+ from datetime import datetime
+ if not hasattr( main, 'apps' ):
+ run.initTest( main )
+
+ description = "High Availability tests - ONOS random failures and Switch random failures with 4x4 Leaf-spine "
+ main.case( description )
+ run.config(main, '4x4', 3)
+ run.installOnos( main )
+ run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
+ # pre-configured routing and bridging test
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main )
+ for i in range(0, main.failures):
+ onosToKill = randint(0, (main.numCtrls-1))
+ switchToKill = randint(0, 3)
+ run.killOnos( main, [ onosToKill ], '8', '32', '2' )
+ run.pingAll( main, 'CASE8_ONOS_Failure%d' % (i+1) )
+ run.killSwitch( main, main.spines[switchToKill]['name'], switches='7', links='24' )
+ time.sleep( main.switchSleep )
+ run.pingAll( main, "CASE8_SWITCH_Failure%d" % (i+1) )
+ run.recoverSwitch( main, main.spines[switchToKill]['name'], switches='8', links='32' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, "CASE8_SWITCH_Recovery%d" % (i+1) )
+ run.recoverOnos( main, [ onosToKill ], '8', '32', '3' )
+ run.checkFlows( main, minFlowCount=350 )
+ run.pingAll( main, 'CASE8_ONOS_Recovery%d' % (i+1) )
+ run.cleanup( main )
\ No newline at end of file
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
new file mode 100644
index 0000000..bf9b6be
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/SRHighAvailability.topo
@@ -0,0 +1,90 @@
+<TOPOLOGY>
+ <COMPONENT>
+
+ <ONOSbench>
+ <host>localhost</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosDriver</type>
+ <connect_order>1</connect_order>
+ <COMPONENTS>
+ <nodes>1</nodes>
+ </COMPONENTS>
+ </ONOSbench>
+
+ <ONOScli1>
+ <host>OC1</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>2</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli1>
+
+ <ONOScli2>
+ <host>OC2</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>3</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli2>
+
+ <ONOScli3>
+ <host>OC3</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>OnosCliDriver</type>
+ <connect_order>4</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOScli3>
+
+ <ONOSrest1>
+ <host>OC1</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>5</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest1>
+
+ <ONOSrest2>
+ <host>OC2</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>6</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest2>
+
+ <ONOSrest3>
+ <host>OC3</host>
+ <port>8181</port>
+ <user>onos</user>
+ <password>rocks</password>
+ <type>OnosRestDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ </COMPONENTS>
+ </ONOSrest3>
+
+ <Mininet1>
+ <host>OCN</host>
+ <user>sdn</user>
+ <password>rocks</password>
+ <type>MininetCliDriver</type>
+ <connect_order>7</connect_order>
+ <COMPONENTS>
+ <home>~/mininet/custom/</home>
+ </COMPONENTS>
+ </Mininet1>
+
+ </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/__init__.py b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/USECASE/SegmentRouting/SRHighAvailability/__init__.py
diff --git a/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.py b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.py
index b388a64..0bef93e 100755
--- a/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.py
+++ b/TestON/tests/USECASE/SegmentRouting/SRLinkFailure/SRLinkFailure.py
@@ -35,7 +35,7 @@
run.killLink( main, 'spine101', 'leaf2', switches='4', links='6' )
run.pingAll( main, "CASE1_Failure" )
run.restoreLink( main, 'spine101', 'leaf2', 'of:0000000000000101',
- 'of:0000000000000002', '2', '1', '4', '8' )
+ 'of:0000000000000002', '2', '3', '4', '8' )
run.pingAll( main, "CASE1_Recovery" )
# TODO Dynamic config of hosts in subnet
# TODO Dynamic config of host not in subnet
@@ -69,7 +69,7 @@
run.killLink( main, 'spine101', 'leaf2', switches='8', links='30' )
run.pingAll( main, "CASE2_Failure" )
run.restoreLink( main, 'spine101', 'leaf2', 'of:0000000000000101',
- 'of:0000000000000002', '2', '1', '8', '32' )
+ 'of:0000000000000002', '2', '3', '8', '32' )
run.pingAll( main, "CASE2_Recovery" )
# TODO Dynamic config of hosts in subnet
# TODO Dynamic config of host not in subnet
@@ -103,7 +103,7 @@
run.killLink( main, 'spine101', 'leaf2', switches='4', links='6' )
run.pingAll( main, "CASE3_Failure" )
run.restoreLink( main, 'spine101', 'leaf2', 'of:0000000000000101',
- 'of:0000000000000002', '2', '1', '4', '8' )
+ 'of:0000000000000002', '2', '3', '4', '8' )
run.pingAll( main, "CASE3_Recovery" )
# TODO Dynamic config of hosts in subnet
# TODO Dynamic config of host not in subnet
@@ -137,7 +137,7 @@
run.killLink( main, 'spine101', 'leaf2', switches='8', links='30' )
run.pingAll( main, "CASE2_Failure" )
run.restoreLink( main, 'spine101', 'leaf2', 'of:0000000000000101',
- 'of:0000000000000002', '2', '1', '8', '32' )
+ 'of:0000000000000002', '2', '3', '8', '32' )
run.pingAll( main, "CASE2_Recovery" )
# TODO Dynamic config of hosts in subnet
# TODO Dynamic config of host not in subnet
diff --git a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
index 61c12ad..7ef28e6 100755
--- a/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
+++ b/TestON/tests/USECASE/SegmentRouting/dependencies/Testcaselib.py
@@ -7,6 +7,9 @@
class Testcaselib:
+
+ useSSH=False
+
@staticmethod
def initTest( main ):
"""
@@ -98,7 +101,7 @@
apps,
tempOnosIp,
onosUser,
- useSSH=True )
+ useSSH=Testcaselib.useSSH )
cellResult = main.ONOSbench.setCell( "temp" )
verifyResult = main.ONOSbench.verifyCell( )
stepResult = cellResult and verifyResult
@@ -125,15 +128,16 @@
actual=stepResult,
onpass="Successfully installed ONOS package",
onfail="Failed to install ONOS package" )
- for i in range( main.numCtrls ):
- onosInstallResult = onosInstallResult and \
- main.ONOSbench.onosSecureSSH(
- node=main.ONOSip[ i ] )
- stepResult = onosInstallResult
- utilities.assert_equals( expect=main.TRUE,
- actual=stepResult,
- onpass="Successfully secure SSH",
- onfail="Failed to secure SSH" )
+ if Testcaselib.useSSH:
+ for i in range( main.numCtrls ):
+ onosInstallResult = onosInstallResult and \
+ main.ONOSbench.onosSecureSSH(
+ node=main.ONOSip[ i ] )
+ stepResult = onosInstallResult
+ utilities.assert_equals( expect=main.TRUE,
+ actual=stepResult,
+ onpass="Successfully secure SSH",
+ onfail="Failed to secure SSH" )
main.step( "Starting ONOS service" )
stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE,
for i in range( main.numCtrls ):
@@ -214,6 +218,51 @@
main.exit( )
@staticmethod
+ def config(main, cfgName, numCtrls):
+ main.spines = []
+
+ main.failures = int(main.params[ 'failures' ])
+ main.cfgName = cfgName
+ main.numCtrls = numCtrls
+
+ if main.cfgName == '2x2' :
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine1' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid1' ]
+ main.spines.append(spine)
+
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine2' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid2' ]
+ main.spines.append(spine)
+
+ elif main.cfgName == '4x4' :
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine1' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid1' ]
+ main.spines.append(spine)
+
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine2' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid2' ]
+ main.spines.append(spine)
+
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine3' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid3' ]
+ main.spines.append(spine)
+
+ spine = {}
+ spine[ 'name' ] = main.params['switches'][ 'spine4' ]
+ spine[ 'dpid' ] = main.params['switches'][ 'spinedpid4' ]
+ main.spines.append(spine)
+
+ else :
+ main.log.error( "Configuration failed!" )
+ main.cleanup( )
+ main.exit( )
+
+ @staticmethod
def checkFlows( main, minFlowCount, dumpflows=True ):
main.step(
" Check whether the flow count is bigger than %s" % minFlowCount )
@@ -282,6 +331,7 @@
main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
main.step( "Kill link between %s and %s" % (end1, end2) )
LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
+ LinkDown = main.Mininet1.link( END2=end1, END1=end2, OPTION="down" )
main.log.info(
"Waiting %s seconds for link down to be discovered" % main.linkSleep )
time.sleep( main.linkSleep )
@@ -317,8 +367,12 @@
main.log.info(
"Waiting %s seconds for link up to be discovered" % main.linkSleep )
time.sleep( main.linkSleep )
- main.CLIs[ main.active ].portstate( dpid=dpid1, port=port1 )
- main.CLIs[ main.active ].portstate( dpid=dpid2, port=port2 )
+
+ for i in range(0, main.numCtrls):
+ onosIsUp = main.ONOSbench.isup( main.ONOSip[ i ] )
+ if onosIsUp == main.TRUE:
+ main.CLIs[ i ].portstate( dpid=dpid1, port=port1 )
+ main.CLIs[ i ].portstate( dpid=dpid2, port=port2 )
time.sleep( main.linkSleep )
result = main.CLIs[ main.active ].checkStatus( numoswitch=switches,
@@ -398,7 +452,9 @@
switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
"""
+
main.step( "Killing ONOS instance" )
+
for i in nodes:
killResult = main.ONOSbench.onosDie( main.CLIs[ i ].ip_address )
utilities.assert_equals( expect=main.TRUE, actual=killResult,
@@ -407,7 +463,32 @@
if i == main.active:
main.active = (i + 1) % main.numCtrls
time.sleep( 12 )
+
if len( nodes ) < main.numCtrls:
+
+ nodesToCheck = []
+ for x in range(0, main.numCtrls):
+ if x not in nodes:
+ nodesToCheck.append(x)
+ nodeResults = utilities.retry( Testcaselib.nodesCheck,
+ False,
+ args=[nodesToCheck],
+ attempts=5,
+ sleep=10 )
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+
+ if not nodeResults:
+ for i in nodes:
+ cli = main.CLIs[i]
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ cli.name,
+ cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.log.error( "Failed to kill ONOS, stopping test" )
+ main.cleanup()
+ main.exit()
+
topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
main.FALSE,
kwargs={ 'numoswitch': switches,
@@ -447,6 +528,26 @@
onfail="ONOS CLI is not ready" )
main.active = i if main.active == -1 else main.active
+ main.step( "Checking ONOS nodes" )
+ nodeResults = utilities.retry( Testcaselib.nodesCheck,
+ False,
+ args=[nodes],
+ attempts=5,
+ sleep=10 )
+ utilities.assert_equals( expect=True, actual=nodeResults,
+ onpass="Nodes check successful",
+ onfail="Nodes check NOT successful" )
+
+ if not nodeResults:
+ for i in nodes:
+ cli = main.CLIs[i]
+ main.log.debug( "{} components not ACTIVE: \n{}".format(
+ cli.name,
+ cli.sendline( "scr:list | grep -v ACTIVE" ) ) )
+ main.log.error( "Failed to start ONOS, stopping test" )
+ main.cleanup()
+ main.exit()
+
topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
main.FALSE,
kwargs={ 'numoswitch': switches,
@@ -541,3 +642,39 @@
main.RESTs[ main.active ].removeNetCfg( subjectClass="apps",
subjectKey="org.onosproject.segmentrouting",
configKey="xconnect" )
+ @staticmethod
+ def nodesCheck( nodes ):
+ nodesOutput = []
+ results = True
+ threads = []
+ for i in nodes:
+ t = main.Thread( target=main.CLIs[i].nodes,
+ name="nodes-" + str( i ),
+ args=[ ] )
+ threads.append( t )
+ t.start()
+
+ for t in threads:
+ t.join()
+ nodesOutput.append( t.result )
+ ips = [ main.ONOSip[ node ] for node in nodes ]
+ ips.sort()
+ for i in nodesOutput:
+ try:
+ current = json.loads( i )
+ activeIps = []
+ currentResult = False
+ for node in current:
+ if node['state'] == 'READY':
+ activeIps.append( node['ip'] )
+ currentResult = True
+ for ip in ips:
+ if ip not in activeIps:
+ currentResult = False
+ break
+ except ( ValueError, TypeError ):
+ main.log.error( "Error parsing nodes output" )
+ main.log.warn( repr( i ) )
+ currentResult = False
+ results = results and currentResult
+ return results