Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 1 | import os
|
| 2 | import imp
|
| 3 | import time
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 4 | import json
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 5 | import urllib
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 6 | from core import utilities
|
| 7 |
|
| 8 |
|
| 9 | class Testcaselib:
|
| 10 | @staticmethod
|
| 11 | def initTest( main ):
|
| 12 | """
|
| 13 | - Construct tests variables
|
| 14 | - GIT ( optional )
|
| 15 | - Checkout ONOS master branch
|
| 16 | - Pull latest ONOS code
|
| 17 | - Building ONOS ( optional )
|
| 18 | - Install ONOS package
|
| 19 | - Build ONOS package
|
| 20 | """
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 21 | main.step( "Constructing test variables" )
|
| 22 | # Test variables
|
| 23 | main.cellName = main.params[ 'ENV' ][ 'cellName' ]
|
| 24 | main.apps = main.params[ 'ENV' ][ 'cellApps' ]
|
| 25 | main.diff = main.params[ 'ENV' ][ 'diffApps' ]
|
| 26 | gitBranch = main.params[ 'GIT' ][ 'branch' ]
|
| 27 | main.path = os.path.dirname( main.testFile )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 28 | main.dependencyPath = main.path + "/../dependencies/"
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 29 | main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
|
| 30 | wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 31 | main.scale = (main.params[ 'SCALE' ][ 'size' ]).split( "," )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 32 | main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 33 | # main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 34 | main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 35 | main.cellData = { } # for creating cell file
|
| 36 | main.CLIs = [ ]
|
| 37 | main.ONOSip = [ ]
|
| 38 | main.RESTs = [ ]
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 39 |
|
| 40 | # Assigning ONOS cli handles to a list
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 41 | for i in range( 1, main.maxNodes + 1 ):
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 42 | main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 43 | main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 44 | main.ONOSip.append( main.CLIs[ i - 1 ].ip_address )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 45 | # -- INIT SECTION, ONLY RUNS ONCE -- #
|
| 46 | main.startUp = imp.load_source( wrapperFile1,
|
| 47 | main.dependencyPath +
|
| 48 | wrapperFile1 +
|
| 49 | ".py" )
|
| 50 |
|
| 51 | copyResult1 = main.ONOSbench.scp( main.Mininet1,
|
| 52 | main.dependencyPath +
|
| 53 | main.topology,
|
| 54 | main.Mininet1.home,
|
| 55 | direction="to" )
|
| 56 | if main.CLIs:
|
| 57 | stepResult = main.TRUE
|
| 58 | else:
|
| 59 | main.log.error( "Did not properly created list of ONOS CLI handle" )
|
| 60 | stepResult = main.FALSE
|
| 61 |
|
| 62 | utilities.assert_equals( expect=main.TRUE,
|
| 63 | actual=stepResult,
|
| 64 | onpass="Successfully construct " +
|
| 65 | "test variables ",
|
| 66 | onfail="Failed to construct test variables" )
|
| 67 |
|
| 68 | @staticmethod
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 69 | def installOnos( main, vlanCfg=True ):
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 70 | """
|
| 71 | - Set up cell
|
| 72 | - Create cell file
|
| 73 | - Set cell file
|
| 74 | - Verify cell file
|
| 75 | - Kill ONOS process
|
| 76 | - Uninstall ONOS cluster
|
| 77 | - Verify ONOS start up
|
| 78 | - Install ONOS cluster
|
| 79 | - Connect to cli
|
| 80 | """
|
| 81 | # main.scale[ 0 ] determines the current number of ONOS controller
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 82 | apps = main.apps
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 83 | if main.diff:
|
| 84 | apps = main.apps + "," + main.diff
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 85 | else:
|
| 86 | main.log.error( "App list is empty" )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 87 | print "NODE COUNT = ", main.numCtrls
|
| 88 | print main.ONOSip
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 89 | tempOnosIp = [ ]
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 90 | main.dynamicHosts = [ 'in1', 'out1' ]
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 91 | for i in range( main.numCtrls ):
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 92 | tempOnosIp.append( main.ONOSip[ i ] )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 93 | onosUser = main.params[ 'ENV' ][ 'cellUser' ]
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 94 | main.step( "Create and Apply cell file" )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 95 | main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
|
| 96 | "temp",
|
| 97 | main.Mininet1.ip_address,
|
| 98 | apps,
|
| 99 | tempOnosIp,
|
Pier | 88189b6 | 2016-09-07 17:01:53 -0700 | [diff] [blame] | 100 | onosUser,
|
| 101 | useSSH=True )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 102 | cellResult = main.ONOSbench.setCell( "temp" )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 103 | verifyResult = main.ONOSbench.verifyCell( )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 104 | stepResult = cellResult and verifyResult
|
| 105 | utilities.assert_equals( expect=main.TRUE,
|
| 106 | actual=stepResult,
|
| 107 | onpass="Successfully applied cell to " + \
|
| 108 | "environment",
|
| 109 | onfail="Failed to apply cell to environment " )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 110 | # kill off all onos processes
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 111 | main.log.info( "Safety check, killing all ONOS processes" +
|
| 112 | " before initiating environment setup" )
|
| 113 | for i in range( main.maxNodes ):
|
| 114 | main.ONOSbench.onosDie( main.ONOSip[ i ] )
|
| 115 | main.step( "Create and Install ONOS package" )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 116 | packageResult = main.ONOSbench.onosPackage( )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 117 |
|
| 118 | onosInstallResult = main.TRUE
|
| 119 | for i in range( main.numCtrls ):
|
| 120 | onosInstallResult = onosInstallResult and \
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 121 | main.ONOSbench.onosInstall(
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 122 | node=main.ONOSip[ i ] )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 123 | stepResult = onosInstallResult
|
| 124 | utilities.assert_equals( expect=main.TRUE,
|
| 125 | actual=stepResult,
|
| 126 | onpass="Successfully installed ONOS package",
|
| 127 | onfail="Failed to install ONOS package" )
|
Pier | 88189b6 | 2016-09-07 17:01:53 -0700 | [diff] [blame] | 128 | for i in range( main.numCtrls ):
|
| 129 | onosInstallResult = onosInstallResult and \
|
| 130 | main.ONOSbench.onosSecureSSH(
|
| 131 | node=main.ONOSip[ i ] )
|
| 132 | stepResult = onosInstallResult
|
| 133 | utilities.assert_equals( expect=main.TRUE,
|
| 134 | actual=stepResult,
|
| 135 | onpass="Successfully secure SSH",
|
| 136 | onfail="Failed to secure SSH" )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 137 | main.step( "Starting ONOS service" )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 138 | stopResult, startResult, onosIsUp = main.TRUE, main.TRUE, main.TRUE,
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 139 | for i in range( main.numCtrls ):
|
| 140 | onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
|
| 141 | if onosIsUp == main.TRUE:
|
| 142 | main.log.report( "ONOS instance is up and ready" )
|
| 143 | else:
|
| 144 | main.log.report( "ONOS instance may not be up, stop and " +
|
| 145 | "start ONOS again " )
|
| 146 | for i in range( main.numCtrls ):
|
| 147 | stopResult = stopResult and \
|
| 148 | main.ONOSbench.onosStop( main.ONOSip[ i ] )
|
| 149 | for i in range( main.numCtrls ):
|
| 150 | startResult = startResult and \
|
| 151 | main.ONOSbench.onosStart( main.ONOSip[ i ] )
|
| 152 | stepResult = onosIsUp and stopResult and startResult
|
| 153 |
|
| 154 | utilities.assert_equals( expect=main.TRUE,
|
| 155 | actual=stepResult,
|
| 156 | onpass="ONOS service is ready",
|
| 157 | onfail="ONOS service did not start properly" )
|
| 158 | main.step( "Checking if ONOS CLI is ready" )
|
| 159 | for i in range( main.numCtrls ):
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 160 | main.CLIs[ i ].startCellCli( )
|
| 161 | cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
|
| 162 | commandlineTimeout=60,
|
| 163 | onosStartTimeout=100 )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 164 | utilities.assert_equals( expect=main.TRUE,
|
| 165 | actual=cliResult,
|
| 166 | onpass="ONOS CLI is ready",
|
| 167 | onfail="ONOS CLI is not ready" )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 168 | main.active = 0
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 169 | for i in range( 10 ):
|
| 170 | ready = True
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 171 | output = main.CLIs[ main.active ].summary( )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 172 | if not output:
|
| 173 | ready = False
|
| 174 | if ready:
|
| 175 | break
|
| 176 | time.sleep( 10 )
|
| 177 | utilities.assert_equals( expect=True, actual=ready,
|
| 178 | onpass="ONOS summary command succeded",
|
| 179 | onfail="ONOS summary command failed" )
|
| 180 |
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 181 | with open( "%s/json/%s.json" % (
|
| 182 | main.dependencyPath, main.cfgName) ) as cfg:
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 183 | main.RESTs[ main.active ].setNetCfg( json.load( cfg ) )
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 184 | with open( "%s/json/%s.chart" % (
|
| 185 | main.dependencyPath, main.cfgName) ) as chart:
|
| 186 | main.pingChart = json.load( chart )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 187 | if not ready:
|
| 188 | main.log.error( "ONOS startup failed!" )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 189 | main.cleanup( )
|
| 190 | main.exit( )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 191 |
|
| 192 | for i in range( main.numCtrls ):
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 193 | main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.segmentrouting" )
|
| 194 | main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.driver.pipeline" )
|
| 195 | main.CLIs[ i ].logSet( "DEBUG", "org.onosproject.store.group.impl" )
|
| 196 | main.CLIs[ i ].logSet( "DEBUG",
|
| 197 | "org.onosproject.net.flowobjective.impl" )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 198 |
|
| 199 | @staticmethod
|
| 200 | def startMininet( main, topology, args="" ):
|
| 201 | main.step( "Starting Mininet Topology" )
|
| 202 | arg = "--onos %d %s" % (main.numCtrls, args)
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 203 | main.topology = topology
|
| 204 | topoResult = main.Mininet1.startNet(
|
Pier | b76944e | 2016-09-07 15:26:31 -0700 | [diff] [blame] | 205 | topoFile=main.Mininet1.home + main.topology, args=arg )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 206 | stepResult = topoResult
|
| 207 | utilities.assert_equals( expect=main.TRUE,
|
| 208 | actual=stepResult,
|
| 209 | onpass="Successfully loaded topology",
|
| 210 | onfail="Failed to load topology" )
|
| 211 | # Exit if topology did not load properly
|
| 212 | if not topoResult:
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 213 | main.cleanup( )
|
| 214 | main.exit( )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 215 |
|
| 216 | @staticmethod
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 217 | def checkFlows( main, minFlowCount, dumpflows=True ):
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 218 | main.step(
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 219 | " Check whether the flow count is bigger than %s" % minFlowCount )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 220 | count = utilities.retry( main.CLIs[ main.active ].checkFlowCount,
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 221 | main.FALSE,
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 222 | kwargs={ 'min': minFlowCount },
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 223 | attempts=10,
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 224 | sleep=10 )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 225 | utilities.assertEquals( \
|
| 226 | expect=True,
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 227 | actual=(count > 0),
|
| 228 | onpass="Flow count looks correct: " + str( count ),
|
| 229 | onfail="Flow count looks wrong: " + str( count ) )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 230 |
|
| 231 | main.step( "Check whether all flow status are ADDED" )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 232 | flowCheck = utilities.retry( main.CLIs[ main.active ].checkFlowsState,
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 233 | main.FALSE,
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 234 | kwargs={ 'isPENDING': False },
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 235 | attempts=2,
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 236 | sleep=10 )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 237 | utilities.assertEquals( \
|
| 238 | expect=main.TRUE,
|
| 239 | actual=flowCheck,
|
| 240 | onpass="Flow status is correct!",
|
| 241 | onfail="Flow status is wrong!" )
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 242 | if dumpflows:
|
| 243 | main.ONOSbench.dumpFlows( main.ONOSip[ main.active ],
|
| 244 | main.logdir,
|
| 245 | "flowsBefore" + main.cfgName )
|
| 246 | main.ONOSbench.dumpGroups( main.ONOSip[ main.active ],
|
| 247 | main.logdir,
|
| 248 | "groupsBefore" + main.cfgName )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 249 |
|
| 250 | @staticmethod
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 251 | def pingAll( main, tag="", dumpflows=True ):
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 252 | main.log.report( "Check full connectivity" )
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 253 | print main.pingChart
|
| 254 | for entry in main.pingChart.itervalues( ):
|
| 255 | print entry
|
| 256 | hosts, expect = entry[ 'hosts' ], entry[ 'expect' ]
|
| 257 | expect = main.TRUE if expect else main.FALSE
|
| 258 | main.step( "Connectivity for %s %s" % (str( hosts ), tag) )
|
| 259 | pa = main.Mininet1.pingallHosts( hosts )
|
| 260 | utilities.assert_equals( expect=expect, actual=pa,
|
| 261 | onpass="IP connectivity successfully tested",
|
| 262 | onfail="IP connectivity failed" )
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 263 | if dumpflows:
|
| 264 | main.ONOSbench.dumpFlows( main.ONOSip[ main.active ],
|
| 265 | main.logdir, "flowsOn" + tag )
|
| 266 | main.ONOSbench.dumpGroups( main.ONOSip[ main.active ],
|
| 267 | main.logdir, "groupsOn" + tag )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 268 |
|
| 269 | @staticmethod
|
| 270 | def killLink( main, end1, end2, switches, links ):
|
| 271 | """
|
| 272 | end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
|
| 273 | switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
|
| 274 | Kill a link and verify ONOS can see the proper link change
|
| 275 | """
|
| 276 | main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 277 | main.step( "Kill link between %s and %s" % (end1, end2) )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 278 | LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 279 | main.log.info(
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 280 | "Waiting %s seconds for link down to be discovered" % main.linkSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 281 | time.sleep( main.linkSleep )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 282 | topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
|
| 283 | main.FALSE,
|
| 284 | kwargs={ 'numoswitch': switches,
|
| 285 | 'numolink': links },
|
| 286 | attempts=10,
|
| 287 | sleep=main.linkSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 288 | result = topology & LinkDown
|
| 289 | utilities.assert_equals( expect=main.TRUE, actual=result,
|
| 290 | onpass="Link down successful",
|
| 291 | onfail="Failed to turn off link?" )
|
| 292 |
|
| 293 | @staticmethod
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 294 | def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches,
|
| 295 | links ):
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 296 | """
|
| 297 | Params:
|
| 298 | end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
|
| 299 | dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 300 | port1, port2: respective port of the end switches that connects to the link, ex.:'1'
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 301 | switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
|
| 302 | Kill a link and verify ONOS can see the proper link change
|
| 303 | """
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 304 | main.step( "Restore link between %s and %s" % (end1, end2) )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 305 | result = False
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 306 | count = 0
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 307 | while True:
|
Flavio Castro | d1f3689 | 2016-07-06 11:49:11 -0700 | [diff] [blame] | 308 | count += 1
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 309 | main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
|
| 310 | main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 311 | main.log.info(
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 312 | "Waiting %s seconds for link up to be discovered" % main.linkSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 313 | time.sleep( main.linkSleep )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 314 | main.CLIs[ main.active ].portstate( dpid=dpid1, port=port1 )
|
| 315 | main.CLIs[ main.active ].portstate( dpid=dpid2, port=port2 )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 316 | time.sleep( main.linkSleep )
|
| 317 |
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 318 | result = main.CLIs[ main.active ].checkStatus( numoswitch=switches,
|
| 319 | numolink=links )
|
| 320 | if count > 5 or result:
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 321 | break
|
| 322 | utilities.assert_equals( expect=main.TRUE, actual=result,
|
| 323 | onpass="Link up successful",
|
| 324 | onfail="Failed to bring link up" )
|
| 325 |
|
| 326 | @staticmethod
|
| 327 | def killSwitch( main, switch, switches, links ):
|
| 328 | """
|
| 329 | Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 330 | Completely kill a switch and verify ONOS can see the proper change
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 331 | """
|
| 332 | main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
|
| 333 | main.step( "Kill " + switch )
|
| 334 | main.log.info( "Stopping" + switch )
|
| 335 | main.Mininet1.switch( SW=switch, OPTION="stop" )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 336 | # todo make this repeatable
|
| 337 | main.log.info( "Waiting %s seconds for switch down to be discovered" % (
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 338 | main.switchSleep) )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 339 | time.sleep( main.switchSleep )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 340 | topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
|
| 341 | main.FALSE,
|
| 342 | kwargs={ 'numoswitch': switches,
|
| 343 | 'numolink': links },
|
| 344 | attempts=10,
|
| 345 | sleep=main.switchSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 346 | utilities.assert_equals( expect=main.TRUE, actual=topology,
|
| 347 | onpass="Kill switch successful",
|
| 348 | onfail="Failed to kill switch?" )
|
| 349 |
|
| 350 | @staticmethod
|
| 351 | def recoverSwitch( main, switch, switches, links ):
|
| 352 | """
|
| 353 | Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
|
| 354 | Recover a switch and verify ONOS can see the proper change
|
| 355 | """
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 356 | # todo make this repeatable
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 357 | main.step( "Recovering " + switch )
|
| 358 | main.log.info( "Starting" + switch )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 359 | main.Mininet1.switch( SW=switch, OPTION="start" )
|
| 360 | main.log.info( "Waiting %s seconds for switch up to be discovered" % (
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 361 | main.switchSleep) )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 362 | time.sleep( main.switchSleep )
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 363 | topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
|
| 364 | main.FALSE,
|
| 365 | kwargs={ 'numoswitch': switches,
|
| 366 | 'numolink': links },
|
| 367 | attempts=10,
|
| 368 | sleep=main.switchSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 369 | utilities.assert_equals( expect=main.TRUE, actual=topology,
|
| 370 | onpass="Switch recovery successful",
|
| 371 | onfail="Failed to recover switch?" )
|
| 372 |
|
| 373 | @staticmethod
|
| 374 | def cleanup( main ):
|
| 375 | """
|
| 376 | Stop Onos-cluster.
|
| 377 | Stops Mininet
|
| 378 | Copies ONOS log
|
| 379 | """
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 380 | main.Mininet1.stopNet( )
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 381 | main.ONOSbench.scp( main.ONOScli1, "/opt/onos/log/karaf.log",
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 382 | "/tmp/karaf.log", direction="from" )
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 383 | main.ONOSbench.cpLogsToDir( "/tmp/karaf.log", main.logdir,
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 384 | copyFileName="karaf.log." + main.cfgName )
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 385 | for i in range( main.numCtrls ):
|
Flavio Castro | 5608a39 | 2016-06-22 17:02:35 -0700 | [diff] [blame] | 386 | main.ONOSbench.onosStop( main.ONOSip[ i ] )
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 387 |
|
| 388 | @staticmethod
|
| 389 | def killOnos( main, nodes, switches, links, expNodes ):
|
| 390 | """
|
| 391 | Params: nodes, integer array with position of the ONOS nodes in the CLIs array
|
| 392 | switches, links, nodes: number of expected switches, links and nodes after KillOnos, ex.: '4', '6'
|
| 393 | Completely Kill an ONOS instance and verify the ONOS cluster can see the proper change
|
| 394 | """
|
| 395 | main.step( "Killing ONOS instance" )
|
| 396 | for i in nodes:
|
| 397 | killResult = main.ONOSbench.onosDie( main.CLIs[ i ].ip_address )
|
| 398 | utilities.assert_equals( expect=main.TRUE, actual=killResult,
|
| 399 | onpass="ONOS instance Killed",
|
| 400 | onfail="Error killing ONOS instance" )
|
| 401 | if i == main.active:
|
| 402 | main.active = (i + 1) % main.numCtrls
|
| 403 | time.sleep( 12 )
|
| 404 | if len( nodes ) < main.numCtrls:
|
| 405 | topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
|
| 406 | main.FALSE,
|
| 407 | kwargs={ 'numoswitch': switches,
|
| 408 | 'numolink': links,
|
| 409 | 'numoctrl': expNodes },
|
| 410 | attempts=10,
|
| 411 | sleep=12 )
|
| 412 | utilities.assert_equals( expect=main.TRUE, actual=topology,
|
| 413 | onpass="ONOS Instance down successful",
|
| 414 | onfail="Failed to turn off ONOS Instance" )
|
| 415 | else:
|
| 416 | main.active = -1
|
| 417 |
|
| 418 | @staticmethod
|
| 419 | def recoverOnos( main, nodes, switches, links, expNodes ):
|
| 420 | """
|
| 421 | Params: nodes, integer array with position of the ONOS nodes in the CLIs array
|
| 422 | switches, links, nodes: number of expected switches, links and nodes after recoverOnos, ex.: '4', '6'
|
| 423 | Recover an ONOS instance and verify the ONOS cluster can see the proper change
|
| 424 | """
|
| 425 | main.step( "Recovering ONOS instance" )
|
| 426 | [ main.ONOSbench.onosStart( main.CLIs[ i ].ip_address ) for i in nodes ]
|
| 427 | for i in nodes:
|
| 428 | isUp = main.ONOSbench.isup( main.ONOSip[ i ] )
|
| 429 | utilities.assert_equals( expect=main.TRUE, actual=isUp,
|
| 430 | onpass="ONOS service is ready",
|
| 431 | onfail="ONOS service did not start properly" )
|
| 432 | for i in nodes:
|
| 433 | main.step( "Checking if ONOS CLI is ready" )
|
| 434 | main.CLIs[ i ].startCellCli( )
|
| 435 | cliResult = main.CLIs[ i ].startOnosCli( main.ONOSip[ i ],
|
| 436 | commandlineTimeout=60,
|
| 437 | onosStartTimeout=100 )
|
| 438 | utilities.assert_equals( expect=main.TRUE,
|
| 439 | actual=cliResult,
|
| 440 | onpass="ONOS CLI is ready",
|
| 441 | onfail="ONOS CLI is not ready" )
|
| 442 | main.active = i if main.active == -1 else main.active
|
| 443 |
|
| 444 | topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
|
| 445 | main.FALSE,
|
| 446 | kwargs={ 'numoswitch': switches,
|
| 447 | 'numolink': links,
|
| 448 | 'numoctrl': expNodes },
|
| 449 | attempts=10,
|
| 450 | sleep=12 )
|
| 451 | utilities.assert_equals( expect=main.TRUE, actual=topology,
|
| 452 | onpass="ONOS Instance down successful",
|
| 453 | onfail="Failed to turn off ONOS Instance" )
|
Flavio Castro | 519072e | 2016-06-23 13:30:48 -0700 | [diff] [blame] | 454 | for i in range( 10 ):
|
| 455 | ready = True
|
| 456 | output = main.CLIs[ main.active ].summary( )
|
| 457 | if not output:
|
| 458 | ready = False
|
| 459 | if ready:
|
| 460 | break
|
| 461 | time.sleep( 10 )
|
| 462 | utilities.assert_equals( expect=True, actual=ready,
|
| 463 | onpass="ONOS summary command succeded",
|
| 464 | onfail="ONOS summary command failed" )
|
| 465 | if not ready:
|
| 466 | main.log.error( "ONOS startup failed!" )
|
| 467 | main.cleanup( )
|
| 468 | main.exit( )
|
Flavio Castro | ab163ca | 2016-07-07 14:05:00 -0700 | [diff] [blame] | 469 |
|
| 470 | @staticmethod
|
| 471 | def addHostCfg( main ):
|
| 472 | """
|
| 473 | Adds Host Configuration to ONOS
|
| 474 | Updates expected state of the network (pingChart)
|
| 475 | """
|
| 476 | import json
|
| 477 | hostCfg = { }
|
| 478 | with open( main.dependencyPath + "/json/extra.json" ) as template:
|
| 479 | hostCfg = json.load( template )
|
| 480 | main.pingChart[ 'ip' ][ 'hosts' ] += [ 'in1' ]
|
| 481 | main.step( "Pushing new configuration" )
|
| 482 | mac, cfg = hostCfg[ 'hosts' ].popitem( )
|
| 483 | main.RESTs[ main.active ].setNetCfg( cfg[ 'basic' ],
|
| 484 | subjectClass="hosts",
|
| 485 | subjectKey=urllib.quote( mac,
|
| 486 | safe='' ),
|
| 487 | configKey="basic" )
|
| 488 | main.pingChart[ 'ip' ][ 'hosts' ] += [ 'out1' ]
|
| 489 | main.step( "Pushing new configuration" )
|
| 490 | mac, cfg = hostCfg[ 'hosts' ].popitem( )
|
| 491 | main.RESTs[ main.active ].setNetCfg( cfg[ 'basic' ],
|
| 492 | subjectClass="hosts",
|
| 493 | subjectKey=urllib.quote( mac,
|
| 494 | safe='' ),
|
| 495 | configKey="basic" )
|
| 496 | main.pingChart.update( { 'vlan1': { "expect": "True",
|
| 497 | "hosts": [ "olt1", "vsg1" ] } } )
|
| 498 | main.pingChart[ 'vlan5' ][ 'expect' ] = 0
|
| 499 | main.pingChart[ 'vlan10' ][ 'expect' ] = 0
|
| 500 | ports = "[%s,%s]" % (5, 6)
|
| 501 | cfg = '{"of:0000000000000001":[{"vlan":1,"ports":%s,"name":"OLT 1"}]}' % ports
|
| 502 | main.RESTs[ main.active ].setNetCfg( json.loads( cfg ),
|
| 503 | subjectClass="apps",
|
| 504 | subjectKey="org.onosproject.segmentrouting",
|
| 505 | configKey="xconnect" )
|
| 506 |
|
| 507 | @staticmethod
|
| 508 | def delHostCfg( main ):
|
| 509 | """
|
| 510 | Removest Host Configuration from ONOS
|
| 511 | Updates expected state of the network (pingChart)
|
| 512 | """
|
| 513 | import json
|
| 514 | hostCfg = { }
|
| 515 | with open( main.dependencyPath + "/json/extra.json" ) as template:
|
| 516 | hostCfg = json.load( template )
|
| 517 | main.step( "Removing host configuration" )
|
| 518 | main.pingChart[ 'ip' ][ 'expect' ] = 0
|
| 519 | mac, cfg = hostCfg[ 'hosts' ].popitem( )
|
| 520 | main.RESTs[ main.active ].removeNetCfg( subjectClass="hosts",
|
| 521 | subjectKey=urllib.quote(
|
| 522 | mac,
|
| 523 | safe='' ),
|
| 524 | configKey="basic" )
|
| 525 | main.step( "Removing configuration" )
|
| 526 | main.pingChart[ 'ip' ][ 'expect' ] = 0
|
| 527 | mac, cfg = hostCfg[ 'hosts' ].popitem( )
|
| 528 | main.RESTs[ main.active ].removeNetCfg( subjectClass="hosts",
|
| 529 | subjectKey=urllib.quote(
|
| 530 | mac,
|
| 531 | safe='' ),
|
| 532 | configKey="basic" )
|
| 533 | main.step( "Removing vlan configuration" )
|
| 534 | main.pingChart[ 'vlan1' ][ 'expect' ] = 0
|
| 535 | main.RESTs[ main.active ].removeNetCfg( subjectClass="apps",
|
| 536 | subjectKey="org.onosproject.segmentrouting",
|
| 537 | configKey="xconnect" )
|