Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 1 | import os
|
| 2 | import imp
|
| 3 | import time
|
| 4 |
|
| 5 | from core import utilities
|
| 6 |
|
| 7 |
|
| 8 | class Testcaselib:
|
| 9 | @staticmethod
|
| 10 | def initTest( main ):
|
| 11 | """
|
| 12 | - Construct tests variables
|
| 13 | - GIT ( optional )
|
| 14 | - Checkout ONOS master branch
|
| 15 | - Pull latest ONOS code
|
| 16 | - Building ONOS ( optional )
|
| 17 | - Install ONOS package
|
| 18 | - Build ONOS package
|
| 19 | """
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 20 | main.step( "Constructing test variables" )
|
| 21 | # Test variables
|
| 22 | main.cellName = main.params[ 'ENV' ][ 'cellName' ]
|
| 23 | main.apps = main.params[ 'ENV' ][ 'cellApps' ]
|
| 24 | main.diff = main.params[ 'ENV' ][ 'diffApps' ]
|
| 25 | gitBranch = main.params[ 'GIT' ][ 'branch' ]
|
| 26 | main.path = os.path.dirname( main.testFile )
|
| 27 | main.dependencyPath = main.path +"/../dependencies/"
|
| 28 | main.topology = main.params[ 'DEPENDENCY' ][ 'topology' ]
|
| 29 | wrapperFile1 = main.params[ 'DEPENDENCY' ][ 'wrapper1' ]
|
| 30 | main.scale = ( main.params[ 'SCALE' ][ 'size' ] ).split( "," )
|
| 31 | main.maxNodes = int( main.params[ 'SCALE' ][ 'max' ] )
|
| 32 | #main.ONOSport = main.params[ 'CTRL' ][ 'port' ]
|
| 33 | main.startUpSleep = int( main.params[ 'SLEEP' ][ 'startup' ] )
|
| 34 | main.cellData = {} # for creating cell file
|
| 35 | main.CLIs = []
|
| 36 | main.ONOSip = []
|
| 37 |
|
| 38 | # Assigning ONOS cli handles to a list
|
| 39 | for i in range( 1, main.maxNodes + 1 ):
|
| 40 | main.CLIs.append( getattr( main, 'ONOScli' + str( i ) ) )
|
| 41 | main.ONOSip.append( main.CLIs[i-1].ip_address )
|
| 42 | # -- INIT SECTION, ONLY RUNS ONCE -- #
|
| 43 | main.startUp = imp.load_source( wrapperFile1,
|
| 44 | main.dependencyPath +
|
| 45 | wrapperFile1 +
|
| 46 | ".py" )
|
| 47 |
|
| 48 | copyResult1 = main.ONOSbench.scp( main.Mininet1,
|
| 49 | main.dependencyPath +
|
| 50 | main.topology,
|
| 51 | main.Mininet1.home,
|
| 52 | direction="to" )
|
| 53 | if main.CLIs:
|
| 54 | stepResult = main.TRUE
|
| 55 | else:
|
| 56 | main.log.error( "Did not properly created list of ONOS CLI handle" )
|
| 57 | stepResult = main.FALSE
|
| 58 |
|
| 59 | utilities.assert_equals( expect=main.TRUE,
|
| 60 | actual=stepResult,
|
| 61 | onpass="Successfully construct " +
|
| 62 | "test variables ",
|
| 63 | onfail="Failed to construct test variables" )
|
| 64 |
|
| 65 | @staticmethod
|
| 66 | def installOnos( main ):
|
| 67 | """
|
| 68 | - Set up cell
|
| 69 | - Create cell file
|
| 70 | - Set cell file
|
| 71 | - Verify cell file
|
| 72 | - Kill ONOS process
|
| 73 | - Uninstall ONOS cluster
|
| 74 | - Verify ONOS start up
|
| 75 | - Install ONOS cluster
|
| 76 | - Connect to cli
|
| 77 | """
|
| 78 | # main.scale[ 0 ] determines the current number of ONOS controller
|
| 79 | apps=main.apps
|
| 80 | if main.diff:
|
| 81 | apps = main.apps + "," + main.diff
|
| 82 | else: main.log.error( "App list is empty" )
|
| 83 | main.case( "Package and start ONOS using apps:" + apps)
|
| 84 | print "NODE COUNT = ", main.numCtrls
|
| 85 | print main.ONOSip
|
| 86 | tempOnosIp = []
|
| 87 | for i in range( main.numCtrls ):
|
| 88 | tempOnosIp.append( main.ONOSip[i] )
|
| 89 | onosUser = main.params[ 'ENV' ][ 'cellUser' ]
|
| 90 | main.step("Create and Apply cell file")
|
| 91 | main.ONOSbench.createCellFile( main.ONOSbench.ip_address,
|
| 92 | "temp",
|
| 93 | main.Mininet1.ip_address,
|
| 94 | apps,
|
| 95 | tempOnosIp,
|
| 96 | onosUser )
|
| 97 | cellResult = main.ONOSbench.setCell( "temp" )
|
| 98 | verifyResult = main.ONOSbench.verifyCell()
|
| 99 | stepResult = cellResult and verifyResult
|
| 100 | utilities.assert_equals( expect=main.TRUE,
|
| 101 | actual=stepResult,
|
| 102 | onpass="Successfully applied cell to " + \
|
| 103 | "environment",
|
| 104 | onfail="Failed to apply cell to environment " )
|
| 105 | #kill off all onos processes
|
| 106 | main.log.info( "Safety check, killing all ONOS processes" +
|
| 107 | " before initiating environment setup" )
|
| 108 | for i in range( main.maxNodes ):
|
| 109 | main.ONOSbench.onosDie( main.ONOSip[ i ] )
|
| 110 | main.step( "Create and Install ONOS package" )
|
| 111 | main.ONOSbench.handle.sendline( "cp "+main.dependencyPath+"/"+main.cfgName+".json ~/onos/tools/package/config/network-cfg.json")
|
| 112 | packageResult = main.ONOSbench.onosPackage()
|
| 113 |
|
| 114 | onosInstallResult = main.TRUE
|
| 115 | for i in range( main.numCtrls ):
|
| 116 | onosInstallResult = onosInstallResult and \
|
| 117 | main.ONOSbench.onosInstall( node=main.ONOSip[ i ] )
|
| 118 | stepResult = onosInstallResult
|
| 119 | utilities.assert_equals( expect=main.TRUE,
|
| 120 | actual=stepResult,
|
| 121 | onpass="Successfully installed ONOS package",
|
| 122 | onfail="Failed to install ONOS package" )
|
| 123 | main.step( "Starting ONOS service" )
|
| 124 | stopResult,startResult, onosIsUp= main.TRUE, main.TRUE, main.TRUE,
|
| 125 | for i in range( main.numCtrls ):
|
| 126 | onosIsUp = onosIsUp and main.ONOSbench.isup( main.ONOSip[ i ] )
|
| 127 | if onosIsUp == main.TRUE:
|
| 128 | main.log.report( "ONOS instance is up and ready" )
|
| 129 | else:
|
| 130 | main.log.report( "ONOS instance may not be up, stop and " +
|
| 131 | "start ONOS again " )
|
| 132 | for i in range( main.numCtrls ):
|
| 133 | stopResult = stopResult and \
|
| 134 | main.ONOSbench.onosStop( main.ONOSip[ i ] )
|
| 135 | for i in range( main.numCtrls ):
|
| 136 | startResult = startResult and \
|
| 137 | main.ONOSbench.onosStart( main.ONOSip[ i ] )
|
| 138 | stepResult = onosIsUp and stopResult and startResult
|
| 139 |
|
| 140 | utilities.assert_equals( expect=main.TRUE,
|
| 141 | actual=stepResult,
|
| 142 | onpass="ONOS service is ready",
|
| 143 | onfail="ONOS service did not start properly" )
|
| 144 | main.step( "Checking if ONOS CLI is ready" )
|
| 145 | for i in range( main.numCtrls ):
|
| 146 | main.CLIs[i].startCellCli()
|
| 147 | cliResult = main.CLIs[i].startOnosCli( main.ONOSip[ i ],
|
| 148 | commandlineTimeout=60, onosStartTimeout=100 )
|
| 149 | utilities.assert_equals( expect=main.TRUE,
|
| 150 | actual=cliResult,
|
| 151 | onpass="ONOS CLI is ready",
|
| 152 | onfail="ONOS CLI is not ready" )
|
| 153 | main.active=0
|
| 154 | for i in range( 10 ):
|
| 155 | ready = True
|
| 156 | output = main.CLIs[main.active].summary()
|
| 157 | if not output:
|
| 158 | ready = False
|
| 159 | if ready:
|
| 160 | break
|
| 161 | time.sleep( 10 )
|
| 162 | utilities.assert_equals( expect=True, actual=ready,
|
| 163 | onpass="ONOS summary command succeded",
|
| 164 | onfail="ONOS summary command failed" )
|
| 165 |
|
| 166 | if not ready:
|
| 167 | main.log.error( "ONOS startup failed!" )
|
| 168 | main.cleanup()
|
| 169 | main.exit()
|
| 170 |
|
| 171 | for i in range( main.numCtrls ):
|
| 172 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.segmentrouting" )
|
| 173 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.driver.pipeline" )
|
| 174 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.store.group.impl" )
|
| 175 | main.CLIs[i].logSet( "DEBUG", "org.onosproject.net.flowobjective.impl" )
|
| 176 |
|
| 177 | @staticmethod
|
| 178 | def startMininet( main, topology, args="" ):
|
| 179 | main.step( "Starting Mininet Topology" )
|
| 180 | arg = "--onos %d %s" % (main.numCtrls, args)
|
| 181 | main.topology=topology
|
| 182 | topoResult = main.Mininet1.startNet( topoFile= main.dependencyPath + main.topology, args=arg )
|
| 183 | stepResult = topoResult
|
| 184 | utilities.assert_equals( expect=main.TRUE,
|
| 185 | actual=stepResult,
|
| 186 | onpass="Successfully loaded topology",
|
| 187 | onfail="Failed to load topology" )
|
| 188 | # Exit if topology did not load properly
|
| 189 | if not topoResult:
|
| 190 | main.cleanup()
|
| 191 | main.exit()
|
| 192 |
|
| 193 | @staticmethod
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 194 | def checkFlows(main, minFlowCount):
|
| 195 | main.step(" Check whether the flow count is bigger than %s" % minFlowCount)
|
| 196 | count = utilities.retry(main.CLIs[main.active].checkFlowCount,
|
| 197 | main.FALSE,
|
| 198 | kwargs={'min':minFlowCount},
|
| 199 | attempts=10,
|
| 200 | sleep=10)
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 201 | utilities.assertEquals( \
|
| 202 | expect=True,
|
| 203 | actual=(count>0),
|
| 204 | onpass="Flow count looks correct: "+str(count),
|
| 205 | onfail="Flow count looks wrong: "+str(count) )
|
| 206 |
|
| 207 | main.step( "Check whether all flow status are ADDED" )
|
| 208 | flowCheck = utilities.retry( main.CLIs[main.active].checkFlowsState,
|
| 209 | main.FALSE,
|
| 210 | kwargs={'isPENDING':False},
|
| 211 | attempts=10,
|
| 212 | sleep=10)
|
| 213 | utilities.assertEquals( \
|
| 214 | expect=main.TRUE,
|
| 215 | actual=flowCheck,
|
| 216 | onpass="Flow status is correct!",
|
| 217 | onfail="Flow status is wrong!" )
|
| 218 | main.ONOSbench.dumpFlows( main.ONOSip[main.active],
|
| 219 | main.logdir, "flowsBefore" + main.cfgName)
|
| 220 | main.ONOSbench.dumpGroups( main.ONOSip[0],
|
| 221 | main.logdir, "groupsBefore" + main.cfgName)
|
| 222 |
|
| 223 | @staticmethod
|
| 224 | def pingAll( main, tag=""):
|
| 225 | main.log.report( "Check full connectivity" )
|
| 226 | main.step("Check full connectivity %s" %tag)
|
| 227 | pa = main.Mininet1.pingall()
|
| 228 | utilities.assert_equals( expect=main.TRUE, actual=pa,
|
| 229 | onpass="Full connectivity successfully tested",
|
| 230 | onfail="Full connectivity failed" )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 231 | main.ONOSbench.dumpFlows( main.ONOSip[main.active],
|
| 232 | main.logdir, "flowsOn" + tag)
|
| 233 | main.ONOSbench.dumpGroups( main.ONOSip[main.active],
|
| 234 | main.logdir, "groupsOn" + tag)
|
| 235 |
|
| 236 | @staticmethod
|
| 237 | def killLink( main, end1, end2, switches, links ):
|
| 238 | """
|
| 239 | end1,end2: identify the switches, ex.: 'leaf1', 'spine1'
|
| 240 | switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
|
| 241 | Kill a link and verify ONOS can see the proper link change
|
| 242 | """
|
| 243 | main.linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
|
| 244 | main.step( "Kill link between %s and %s" %(end1, end2))
|
| 245 | LinkDown = main.Mininet1.link( END1=end1, END2=end2, OPTION="down" )
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 246 | main.log.info( "Waiting %s seconds for link down to be discovered" % main.linkSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 247 | time.sleep( main.linkSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 248 | topology = utilities.retry( main.CLIs[main.active].checkStatus,
|
| 249 | main.FALSE,
|
| 250 | kwargs={'numoswitch':switches, 'numolink':links},
|
| 251 | attempts=10,
|
| 252 | sleep=main.linkSleep)
|
| 253 | result = topology & LinkDown
|
| 254 | utilities.assert_equals( expect=main.TRUE, actual=result,
|
| 255 | onpass="Link down successful",
|
| 256 | onfail="Failed to turn off link?" )
|
| 257 |
|
| 258 | @staticmethod
|
| 259 | def restoreLink( main, end1, end2, dpid1, dpid2, port1, port2, switches, links ):
|
| 260 | """
|
| 261 | Params:
|
| 262 | end1,end2: identify the end switches, ex.: 'leaf1', 'spine1'
|
| 263 | dpid1, dpid2: dpid of the end switches respectively, ex.: 'of:0000000000000002'
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 264 | port1, port2: respective port of the end switches that connects to the link, ex.:'1'
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 265 | switches, links: number of expected switches and links after linkDown, ex.: '4', '6'
|
| 266 | Kill a link and verify ONOS can see the proper link change
|
| 267 | """
|
| 268 | main.step( "Restore link between %s and %s" %( end1, end2 ) )
|
| 269 | result = False
|
| 270 | count=0
|
| 271 | while True:
|
| 272 | count+=0
|
| 273 | main.Mininet1.link( END1=end1, END2=end2, OPTION="up" )
|
| 274 | main.Mininet1.link( END2=end1, END1=end2, OPTION="up" )
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 275 | main.log.info( "Waiting %s seconds for link up to be discovered" % main.linkSleep )
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 276 | time.sleep( main.linkSleep )
|
| 277 | main.CLIs[main.active].portstate( dpid=dpid1, port=port1 )
|
| 278 | main.CLIs[main.active].portstate( dpid=dpid2, port=port2 )
|
| 279 | time.sleep( main.linkSleep )
|
| 280 |
|
| 281 | result = main.CLIs[main.active].checkStatus( numoswitch=switches, numolink=links )
|
| 282 | if count>5 or result:
|
| 283 | break
|
| 284 | utilities.assert_equals( expect=main.TRUE, actual=result,
|
| 285 | onpass="Link up successful",
|
| 286 | onfail="Failed to bring link up" )
|
| 287 |
|
| 288 | @staticmethod
|
| 289 | def killSwitch( main, switch, switches, links ):
|
| 290 | """
|
| 291 | Params: switches, links: number of expected switches and links after SwitchDown, ex.: '4', '6'
|
Flavio Castro | 02b4363 | 2016-06-20 17:07:27 -0700 | [diff] [blame] | 292 | Completely kill a switch and verify ONOS can see the proper change
|
Flavio Castro | 0a05b8d | 2016-06-16 17:26:51 -0700 | [diff] [blame] | 293 | """
|
| 294 | main.switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
|
| 295 | main.step( "Kill " + switch )
|
| 296 | main.log.info( "Stopping" + switch )
|
| 297 | main.Mininet1.switch( SW=switch, OPTION="stop" )
|
| 298 | main.log.info( "Waiting %s seconds for switch down to be discovered" % ( main.switchSleep ) )
|
| 299 | time.sleep( main.switchSleep )
|
| 300 | topology = utilities.retry( main.CLIs[main.active].checkStatus,
|
| 301 | main.FALSE,
|
| 302 | kwargs={'numoswitch':switches, 'numolink':links},
|
| 303 | attempts=10,
|
| 304 | sleep=main.switchSleep)
|
| 305 | utilities.assert_equals( expect=main.TRUE, actual=topology,
|
| 306 | onpass="Kill switch successful",
|
| 307 | onfail="Failed to kill switch?" )
|
| 308 |
|
| 309 | @staticmethod
|
| 310 | def recoverSwitch( main, switch, switches, links ):
|
| 311 | """
|
| 312 | Params: switches, links: number of expected switches and links after SwitchUp, ex.: '4', '6'
|
| 313 | Recover a switch and verify ONOS can see the proper change
|
| 314 | """
|
| 315 | main.step( "Recovering " + switch )
|
| 316 | main.log.info( "Starting" + switch )
|
| 317 | main.Mininet1.switch( SW=switch, OPTION="start")
|
| 318 | main.log.info( "Waiting %s seconds for switch up to be discovered" %(main.switchSleep))
|
| 319 | time.sleep( main.switchSleep )
|
| 320 | topology = utilities.retry( main.CLIs[main.active].checkStatus,
|
| 321 | main.FALSE,
|
| 322 | kwargs={'numoswitch':switches, 'numolink':links},
|
| 323 | attempts=10,
|
| 324 | sleep=main.switchSleep)
|
| 325 | utilities.assert_equals( expect=main.TRUE, actual=topology,
|
| 326 | onpass="Switch recovery successful",
|
| 327 | onfail="Failed to recover switch?" )
|
| 328 |
|
| 329 | @staticmethod
|
| 330 | def cleanup( main ):
|
| 331 | """
|
| 332 | Stop Onos-cluster.
|
| 333 | Stops Mininet
|
| 334 | Copies ONOS log
|
| 335 | """
|
| 336 | main.Mininet1.stopNet()
|
| 337 | main.ONOSbench.scp( main.ONOScli1 ,"/opt/onos/log/karaf.log",
|
| 338 | "/tmp/karaf.log", direction="from" )
|
| 339 | main.ONOSbench.cpLogsToDir("/tmp/karaf.log",main.logdir,
|
| 340 | copyFileName="karaf.log."+main.cfgName)
|
| 341 | for i in range(main.numCtrls):
|
| 342 | main.ONOSbench.onosStop( main.ONOSip[i] )
|