Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 1 | import os
|
| 2 | import imp
|
| 3 | import time
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 4 | import json
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 5 |
|
| 6 | from core import utilities
|
| 7 |
|
| 8 |
|
| 9 | class Testcaselib:
|
| 10 | @staticmethod
|
| 11 | def initTest( main ):
|
| 12 | """
|
| 13 | - Construct tests variables
|
| 14 | - GIT ( optional )
|
| 15 | - Checkout ONOS master branch
|
| 16 | - Pull latest ONOS code
|
| 17 | - Building ONOS ( optional )
|
| 18 | - Install ONOS package
|
| 19 | - Build ONOS package
|
| 20 | """
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 21 | main.step( "Constructing test variables" )
|
| 22 | # Test variables
|
| 23 | main.cellName = main.params[ 'ENV' ][ 'cellName' ]
|
| 24 | main.apps = main.params[ 'ENV' ][ 'cellApps' ]
|
| 25 | main.diff = main.params[ 'ENV' ][ 'diffApps' ]
|
| 26 | gitBranch = main.params[ 'GIT' ][ 'branch' ]
|
| 27 | main.path = os.path.dirname( main.testFile )
|
| 28 | main.dependencyPath = main.path +"/../dependencies/"
|
| 29 | main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
|
| 30 | wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
|
| 31 | main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
|
| 32 | main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
|
| 33 | #main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
|
| 34 | main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
|
| 35 | main.cellData = {} # for creating cell file
|
| 36 | main.CLIs = []
|
| 37 | main.ONOSip = []
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 38 | main.RESTs= []
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 39 |
|
| 40 | # Assigning ONOS cli handles to a list
|
| 41 | for i in range( 1, main.maxNodes + 1 ):
|
| 42 | main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 43 | main.RESTs.append( getattr( main, 'ONOSrest' + str( i ) ) )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 44 | main.ONOSip.append( main.CLIs[i-1].ip_address )
|
| 45 | # -- INIT SECTION, ONLY RUNS ONCE -- #
|
| 46 | main.startUp = imp.load_source( wrapperFile1,
|
| 47 | main.dependencyPath +
|
| 48 | wrapperFile1 +
|
| 49 | ".py" )
|
| 50 |
|
| 51 | copyResult1 = main.ONOSbench.scp( main.Mininet1,
|
| 52 | main.dependencyPath +
|
| 53 | main.topology,
|
| 54 | main.Mininet1.home,
|
| 55 | direction="to" )
|
| 56 | if main.CLIs:
|
| 57 | stepResult = main.TRUE
|
| 58 | else:
|
| 59 | main.log.error( "Did not properly created list of ONOS CLI handle" )
|
| 60 | stepResult = main.FALSE
|
| 61 |
|
| 62 | utilities.assert_equals( expect=main.TRUE,
|
| 63 | actual=stepResult,
|
| 64 | onpass="Successfully construct " +
|
| 65 | "test variables ",
|
| 66 | onfail="Failed to construct test variables" )
|
| 67 |
|
| 68 | @staticmethod
|
| 69 | def installOnos( main ):
|
| 70 | """
|
| 71 | - Set up cell
|
| 72 | - Create cell file
|
| 73 | - Set cell file
|
| 74 | - Verify cell file
|
| 75 | - Kill ONOS process
|
| 76 | - Uninstall ONOS cluster
|
| 77 | - Verify ONOS start up
|
| 78 | - Install ONOS cluster
|
| 79 | - Connect to cli
|
| 80 | """
|
| 81 | # main.scale[ 0 ] determines the current number of ONOS controller
|
| 82 | apps=main.apps
|
| 83 | if main.diff:
|
| 84 | apps = main.apps + "," + main.diff
|
| 85 | else: main.log.error( "App list is empty" )
|
| 86 | main.case( "Package and start ONOS using apps:" + apps)
|
| 87 | print "NODE COUNT = ", main.numCtrls
|
| 88 | print main.ONOSip
|
| 89 | tempOnosIp = []
|
| 90 | for i in range( main.numCtrls ):
|
| 91 | tempOnosIp.append( main.ONOSip[i] )
|
| 92 | onosUser = main.params[ 'ENV' ][ 'cellUser' ]
|
| 93 | main.step("Create and Apply cell file")
|
| 94 | main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
|
| 95 | "temp",
|
| 96 | main.Mininet1.ip_address,
|
| 97 | apps,
|
| 98 | tempOnosIp,
|
| 99 | onosUser )
|
| 100 | cellResult = main.ONOSbench.setCell( "temp" )
|
| 101 | verifyResult = main.ONOSbench.verifyCell()
|
| 102 | stepResult = cellResult and verifyResult
|
| 103 | utilities.assert_equals( expect=main.TRUE,
|
| 104 | actual=stepResult,
|
| 105 | onpass="Successfully applied cell to " + \
|
| 106 | "environment",
|
| 107 | onfail="Failed to apply cell to environment " )
|
| 108 | #kill off all onos processes
|
| 109 | main.log.info( "Safety check, killing all ONOS processes" +
|
| 110 | " before initiating environment setup" )
|
| 111 | for i in range( main.maxNodes ):
|
| 112 | main.ONOSbench.onosDie( main.ONOSip[ i ] )
|
| 113 | main.step( "Create and Install ONOS package" )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 114 | packageResult = main.ONOSbench.onosPackage()
|
| 115 |
|
| 116 | onosInstallResult = main.TRUE
|
| 117 | for i in range( main.numCtrls ):
|
| 118 | onosInstallResult = onosInstallResult and \
|
| 119 | main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
|
| 120 | stepResult = onosInstallResult
|
| 121 | utilities.assert_equals( expect=main.TRUE,
|
| 122 | actual=stepResult,
|
| 123 | onpass="Successfully installed ONOS package",
|
| 124 | onfail="Failed to install ONOS package" )
|
| 125 | main.step( "Starting ONOS service" )
|
| 126 | stopResult,startResult, onosIsUp= main.TRUE, main.TRUE, main.TRUE,
|
| 127 | for i in range( main.numCtrls ):
|
| 128 | onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
|
| 129 | if onosIsUp == main.TRUE:
|
| 130 | main.log.report( "ONOS instance is up and ready" )
|
| 131 | else:
|
| 132 | main.log.report( "ONOS instance may not be up, stop and " +
|
| 133 | "start ONOS again " )
|
| 134 | for i in range( main.numCtrls ):
|
| 135 | stopResult = stopResult and \
|
| 136 | main.ONOSbench.onosStop( main.ONOSip[ i ] )
|
| 137 | for i in range( main.numCtrls ):
|
| 138 | startResult = startResult and \
|
| 139 | main.ONOSbench.onosStart( main.ONOSip[ i ] )
|
| 140 | stepResult = onosIsUp and stopResult and startResult
|
| 141 |
|
| 142 | utilities.assert_equals( expect=main.TRUE,
|
| 143 | actual=stepResult,
|
| 144 | onpass="ONOS service is ready",
|
| 145 | onfail="ONOS service did not start properly" )
|
| 146 | main.step( "Checking if ONOS CLI is ready" )
|
| 147 | for i in range( main.numCtrls ):
|
| 148 | main.CLIs[i].startCellCli()
|
| 149 | cliResult = main.CLIs[i].startOnosCli( main.ONOSip[ i ],
|
| 150 | commandlineTimeout=60, onosStartTimeout=100 )
|
| 151 | utilities.assert_equals( expect=main.TRUE,
|
| 152 | actual=cliResult,
|
| 153 | onpass="ONOS CLI is ready",
|
| 154 | onfail="ONOS CLI is not ready" )
|
| 155 | main.active=0
|
| 156 | for i in range( 10 ):
|
| 157 | ready = True
|
| 158 | output = main.CLIs[main.active].summary()
|
| 159 | if not output:
|
| 160 | ready = False
|
| 161 | if ready:
|
| 162 | break
|
| 163 | time.sleep( 10 )
|
| 164 | utilities.assert_equals( expect=True, actual=ready,
|
| 165 | onpass="ONOS summary command succeded",
|
| 166 | onfail="ONOS summary command failed" )
|
| 167 |
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 168 | with open( main.dependencyPath + "/" + main.cfgName + ".json" ) as cfg:
|
| 169 | main.RESTs[main.active].setNetCfg( json.load(cfg) )
|
| 170 |
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 171 | if not ready:
|
| 172 | main.log.error( "ONOS startup failed!" )
|
| 173 | main.cleanup()
|
| 174 | main.exit()
|
| 175 |
|
| 176 | for i in range( main.numCtrls ):
|
| 177 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.segmentrouting" )
|
| 178 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.driver.pipeline" )
|
| 179 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.store.group.impl" )
|
| 180 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.net.flowobjective.impl" )
|
| 181 |
|
| 182 | @staticmethod
|
| 183 | def startMininet( main, topology, args="" ):
|
| 184 | main.step( "Starting Mininet Topology" )
|
| 185 | arg = "--onos %d %s" % (main.numCtrls, args)
|
| 186 | main.topology=topology
|
| 187 | topoResult = main.Mininet1.startNet( topoFile= main.dependencyPath + main.topology, args=arg )
|
| 188 | stepResult = topoResult
|
| 189 | utilities.assert_equals( expect=main.TRUE,
|
| 190 | actual=stepResult,
|
| 191 | onpass="Successfully loaded topology",
|
| 192 | onfail="Failed to load topology" )
|
| 193 | # Exit if topology did not load properly
|
| 194 | if not topoResult:
|
| 195 | main.cleanup()
|
| 196 | main.exit()
|
| 197 |
|
| 198 | @staticmethod
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 199 | def checkFlows(main, minFlowCount):
|
| 200 | main.step(" Check whether the flow count is bigger than %s" % minFlowCount)
|
| 201 | count = utilities.retry(main.CLIs[main.active].checkFlowCount,
|
| 202 | main.FALSE,
|
| 203 | kwargs={'min':minFlowCount},
|
| 204 | attempts=10,
|
| 205 | sleep=10)
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 206 | utilities.assertEquals( \
|
| 207 | expect=True,
|
| 208 | actual=(count>0),
|
| 209 | onpass="Flow count looks correct: "+str(count),
|
| 210 | onfail="Flow count looks wrong: "+str(count) )
|
| 211 |
|
| 212 | main.step( "Check whether all flow status are ADDED" )
|
| 213 | flowCheck = utilities.retry( main.CLIs[main.active].checkFlowsState,
|
| 214 | main.FALSE,
|
| 215 | kwargs={'isPENDING':False},
|
| 216 | attempts=10,
|
| 217 | sleep=10)
|
| 218 | utilities.assertEquals( \
|
| 219 | expect=main.TRUE,
|
| 220 | actual=flowCheck,
|
| 221 | onpass="Flow status is correct!",
|
| 222 | onfail="Flow status is wrong!" )
|
| 223 | main.ONOSbench.dumpFlows( main.ONOSip[main.active],
|
| 224 | main.logdir, "flowsBefore" + main.cfgName)
|
| 225 | main.ONOSbench.dumpGroups( main.ONOSip[0],
|
| 226 | main.logdir, "groupsBefore" + main.cfgName)
|
| 227 |
|
| 228 | @staticmethod
|
| 229 | def pingAll( main, tag=""):
|
| 230 | main.log.report( "Check full connectivity" )
|
| 231 | main.step("Check full connectivity %s" %tag)
|
| 232 | pa = main.Mininet1.pingall()
|
| 233 | utilities.assert_equals( expect=main.TRUE, actual=pa,
|
| 234 | onpass="Full connectivity successfully tested",
|
| 235 | onfail="Full connectivity failed" )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 236 | main.ONOSbench.dumpFlows( main.ONOSip[main.active],
|
| 237 | main.logdir, "flowsOn" + tag)
|
| 238 | main.ONOSbench.dumpGroups( main.ONOSip[main.active],
|
| 239 | main.logdir, "groupsOn" + tag)
|
| 240 |
|
| 241 | @staticmethod
|
| 242 | def killLink( main, end1, end2, switches, links ):
|
| 243 | """
|
| 244 | end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
|
| 245 | switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
|
| 246 | Kill a link and verify ONOS can see the proper link change
|
| 247 | """
|
| 248 | main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
|
| 249 | main.step( "Kill link between %s and %s" %(end1, end2))
|
| 250 | LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 251 | main.log.info( "Waiting %s seconds for link down to be discovered" % main.linkSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 252 | time.sleep( main.linkSleep )
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 253 | topology = utilities.retry( main.CLIs[ main.active ].checkStatus,
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 254 | main.FALSE,
|
| 255 | kwargs={'numoswitch':switches, 'numolink':links},
|
| 256 | attempts=10,
|
| 257 | sleep=main.linkSleep)
|
| 258 | result = topology & LinkDown
|
| 259 | utilities.assert_equals( expect=main.TRUE, actual=result,
|
| 260 | onpass="Link down successful",
|
| 261 | onfail="Failed to turn off link?" )
|
| 262 |
|
| 263 | @staticmethod
|
| 264 | def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches, links ):
|
| 265 | """
|
| 266 | Params:
|
| 267 | end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
|
| 268 | dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 269 | port1, port2: respective port of the end switches that connects to the link, ex.:'1'
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 270 | switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
|
| 271 | Kill a link and verify ONOS can see the proper link change
|
| 272 | """
|
| 273 | main.step( "Restore link between %s and %s" %( end1, end2 ) )
|
| 274 | result = False
|
| 275 | count=0
|
| 276 | while True:
|
| 277 | count+=0
|
| 278 | main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
|
| 279 | main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 280 | main.log.info( "Waiting %s seconds for link up to be discovered" % main.linkSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 281 | time.sleep( main.linkSleep )
|
| 282 | main.CLIs[main.active].portstate( dpid=dpid1, port=port1 )
|
| 283 | main.CLIs[main.active].portstate( dpid=dpid2, port=port2 )
|
| 284 | time.sleep( main.linkSleep )
|
| 285 |
|
| 286 | result = main.CLIs[main.active].checkStatus( numoswitch=switches, numolink=links )
|
| 287 | if count>5 or result:
|
| 288 | break
|
| 289 | utilities.assert_equals( expect=main.TRUE, actual=result,
|
| 290 | onpass="Link up successful",
|
| 291 | onfail="Failed to bring link up" )
|
| 292 |
|
| 293 | @staticmethod
|
| 294 | def killSwitch( main, switch, switches, links ):
|
| 295 | """
|
| 296 | Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 297 | Completely kill a switch and verify ONOS can see the proper change
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 298 | """
|
| 299 | main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
|
| 300 | main.step( "Kill " + switch )
|
| 301 | main.log.info( "Stopping" + switch )
|
| 302 | main.Mininet1.switch( SW=switch, OPTION="stop" )
|
| 303 | main.log.info( "Waiting %s seconds for switch down to be discovered" % ( main.switchSleep ) )
|
| 304 | time.sleep( main.switchSleep )
|
| 305 | topology = utilities.retry( main.CLIs[main.active].checkStatus,
|
| 306 | main.FALSE,
|
| 307 | kwargs={'numoswitch':switches, 'numolink':links},
|
| 308 | attempts=10,
|
| 309 | sleep=main.switchSleep)
|
| 310 | utilities.assert_equals( expect=main.TRUE, actual=topology,
|
| 311 | onpass="Kill switch successful",
|
| 312 | onfail="Failed to kill switch?" )
|
| 313 |
|
| 314 | @staticmethod
|
| 315 | def recoverSwitch( main, switch, switches, links ):
|
| 316 | """
|
| 317 | Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
|
| 318 | Recover a switch and verify ONOS can see the proper change
|
| 319 | """
|
| 320 | main.step( "Recovering " + switch )
|
| 321 | main.log.info( "Starting" + switch )
|
| 322 | main.Mininet1.switch( SW=switch, OPTION="start")
|
| 323 | main.log.info( "Waiting %s seconds for switch up to be discovered" %(main.switchSleep))
|
| 324 | time.sleep( main.switchSleep )
|
| 325 | topology = utilities.retry( main.CLIs[main.active].checkStatus,
|
| 326 | main.FALSE,
|
| 327 | kwargs={'numoswitch':switches, 'numolink':links},
|
| 328 | attempts=10,
|
| 329 | sleep=main.switchSleep)
|
| 330 | utilities.assert_equals( expect=main.TRUE, actual=topology,
|
| 331 | onpass="Switch recovery successful",
|
| 332 | onfail="Failed to recover switch?" )
|
| 333 |
|
| 334 | @staticmethod
|
| 335 | def cleanup( main ):
|
| 336 | """
|
| 337 | Stop Onos-cluster.
|
| 338 | Stops Mininet
|
| 339 | Copies ONOS log
|
| 340 | """
|
| 341 | main.Mininet1.stopNet()
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 342 | main.ONOSbench.scp( main.ONOScli1, "/opt/onos/log/karaf.log",
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 343 | "/tmp/karaf.log", direction="from" )
|
Flavio Castro | 49c218d | 2016-06-21 16:55:57 -0700 | [diff] [blame] | 344 | main.ONOSbench.cpLogsToDir( "/tmp/karaf.log", main.logdir,
|
| 345 | copyFileName="karaf.log."+main.cfgName )
|
| 346 | for i in range( main.numCtrls ):
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 347 | main.ONOSbench.onosStop( main.ONOSip[i] )
|