blob: 590ff569358d6ba71261e73f6d18e3a6e5bd7a24 [file] [log] [blame]
Jon Hall6aec96b2015-01-19 14:49:31 -08001"""
Jon Hallb1290e82014-11-18 16:17:48 -05002Description: This test is to determine if the HA test setup is
3 working correctly. There are no failures so this test should
4 have a 100% pass rate
5
6List of test cases:
7CASE1: Compile ONOS and push it to the test machines
8CASE2: Assign mastership to controllers
9CASE3: Assign intents
10CASE4: Ping across added host intents
11CASE5: Reading state of ONOS
12CASE6: The Failure case. Since this is the Sanity test, we do nothing.
Jon Hall368769f2014-11-19 15:43:35 -080013CASE7: Check state after control plane failure
Jon Hallb1290e82014-11-18 16:17:48 -050014CASE8: Compare topo
15CASE9: Link s3-s28 down
16CASE10: Link s3-s28 up
17CASE11: Switch down
18CASE12: Switch up
19CASE13: Clean up
Jon Hall669173b2014-12-17 11:36:30 -080020CASE14: start election app on all onos nodes
21CASE15: Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -080022"""
Jon Hall8f89dda2015-01-22 16:03:33 -080023
24
Jon Hallb1290e82014-11-18 16:17:48 -050025class HATestSanity:
26
Jon Hall6aec96b2015-01-19 14:49:31 -080027 def __init__( self ):
Jon Hallb1290e82014-11-18 16:17:48 -050028 self.default = ''
29
Jon Hall6aec96b2015-01-19 14:49:31 -080030 def CASE1( self, main ):
31 """
Jon Hallb1290e82014-11-18 16:17:48 -050032 CASE1 is to compile ONOS and push it to the test machines
33
34 Startup sequence:
35 git pull
36 mvn clean install
37 onos-package
38 cell <name>
39 onos-verify-cell
40 NOTE: temporary - onos-remove-raft-logs
41 onos-install -f
42 onos-wait-for-start
Jon Hall6aec96b2015-01-19 14:49:31 -080043 """
44 main.log.report( "ONOS HA Sanity test - initialization" )
45 main.case( "Setting up test environment" )
46 # TODO: save all the timers and output them for plotting
Jon Hallb1290e82014-11-18 16:17:48 -050047
48 # load some vairables from the params file
Jon Hall8f89dda2015-01-22 16:03:33 -080049 PULLCODE = False
Jon Hall6aec96b2015-01-19 14:49:31 -080050 if main.params[ 'Git' ] == 'True':
Jon Hall8f89dda2015-01-22 16:03:33 -080051 PULLCODE = True
Jon Hall529a37f2015-01-28 10:02:00 -080052 gitBranch = main.params[ 'branch' ]
Jon Hall8f89dda2015-01-22 16:03:33 -080053 cellName = main.params[ 'ENV' ][ 'cellName' ]
Jon Hall6aec96b2015-01-19 14:49:31 -080054
55 # set global variables
Jon Hall8f89dda2015-01-22 16:03:33 -080056 global ONOS1Ip
57 global ONOS1Port
58 global ONOS2Ip
59 global ONOS2Port
60 global ONOS3Ip
61 global ONOS3Port
62 global ONOS4Ip
63 global ONOS4Port
64 global ONOS5Ip
65 global ONOS5Port
66 global ONOS6Ip
67 global ONOS6Port
68 global ONOS7Ip
69 global ONOS7Port
70 global numControllers
Jon Hallb1290e82014-11-18 16:17:48 -050071
Jon Hall8f89dda2015-01-22 16:03:33 -080072 ONOS1Ip = main.params[ 'CTRL' ][ 'ip1' ]
73 ONOS1Port = main.params[ 'CTRL' ][ 'port1' ]
74 ONOS2Ip = main.params[ 'CTRL' ][ 'ip2' ]
75 ONOS2Port = main.params[ 'CTRL' ][ 'port2' ]
76 ONOS3Ip = main.params[ 'CTRL' ][ 'ip3' ]
77 ONOS3Port = main.params[ 'CTRL' ][ 'port3' ]
78 ONOS4Ip = main.params[ 'CTRL' ][ 'ip4' ]
79 ONOS4Port = main.params[ 'CTRL' ][ 'port4' ]
80 ONOS5Ip = main.params[ 'CTRL' ][ 'ip5' ]
81 ONOS5Port = main.params[ 'CTRL' ][ 'port5' ]
82 ONOS6Ip = main.params[ 'CTRL' ][ 'ip6' ]
83 ONOS6Port = main.params[ 'CTRL' ][ 'port6' ]
84 ONOS7Ip = main.params[ 'CTRL' ][ 'ip7' ]
85 ONOS7Port = main.params[ 'CTRL' ][ 'port7' ]
86 numControllers = int( main.params[ 'num_controllers' ] )
Jon Hallb1290e82014-11-18 16:17:48 -050087
Jon Hall6aec96b2015-01-19 14:49:31 -080088 main.step( "Applying cell variable to environment" )
Jon Hall8f89dda2015-01-22 16:03:33 -080089 cellResult = main.ONOSbench.setCell( cellName )
90 verifyResult = main.ONOSbench.verifyCell()
Jon Hall368769f2014-11-19 15:43:35 -080091
Jon Hall6aec96b2015-01-19 14:49:31 -080092 # FIXME:this is short term fix
93 main.log.report( "Removing raft logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -080094 main.ONOSbench.onosRemoveRaftLogs()
Jon Hall6aec96b2015-01-19 14:49:31 -080095 main.log.report( "Uninstalling ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -080096 main.ONOSbench.onosUninstall( ONOS1Ip )
97 main.ONOSbench.onosUninstall( ONOS2Ip )
98 main.ONOSbench.onosUninstall( ONOS3Ip )
99 main.ONOSbench.onosUninstall( ONOS4Ip )
100 main.ONOSbench.onosUninstall( ONOS5Ip )
101 main.ONOSbench.onosUninstall( ONOS6Ip )
102 main.ONOSbench.onosUninstall( ONOS7Ip )
Jon Hallb1290e82014-11-18 16:17:48 -0500103
Jon Hall8f89dda2015-01-22 16:03:33 -0800104 cleanInstallResult = main.TRUE
105 gitPullResult = main.TRUE
Jon Hallb1290e82014-11-18 16:17:48 -0500106
Jon Hall97f31752015-02-04 12:01:04 -0800107 main.step( "Starting Mininet" )
108 main.Mininet1.startNet( )
109
Jon Hall6aec96b2015-01-19 14:49:31 -0800110 main.step( "Compiling the latest version of ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800111 if PULLCODE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800112 # TODO Configure branch in params
113 main.step( "Git checkout and pull master" )
Jon Hall529a37f2015-01-28 10:02:00 -0800114 main.ONOSbench.gitCheckout( gitBranch )
Jon Hall8f89dda2015-01-22 16:03:33 -0800115 gitPullResult = main.ONOSbench.gitPull()
Jon Hallb1290e82014-11-18 16:17:48 -0500116
Jon Hall6aec96b2015-01-19 14:49:31 -0800117 main.step( "Using mvn clean & install" )
Jon Hall529a37f2015-01-28 10:02:00 -0800118 cleanInstallResult = main.ONOSbench.cleanInstall()
119 else:
120 main.log.warn( "Did not pull new code so skipping mvn " +
121 "clean install" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800122 main.ONOSbench.getVersion( report=True )
Jon Hallb1290e82014-11-18 16:17:48 -0500123
Jon Hall6aec96b2015-01-19 14:49:31 -0800124 main.step( "Creating ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800125 packageResult = main.ONOSbench.onosPackage()
Jon Hallb1290e82014-11-18 16:17:48 -0500126
Jon Hall6aec96b2015-01-19 14:49:31 -0800127 main.step( "Installing ONOS package" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800128 onos1InstallResult = main.ONOSbench.onosInstall( options="-f",
129 node=ONOS1Ip )
130 onos2InstallResult = main.ONOSbench.onosInstall( options="-f",
131 node=ONOS2Ip )
132 onos3InstallResult = main.ONOSbench.onosInstall( options="-f",
133 node=ONOS3Ip )
134 onos4InstallResult = main.ONOSbench.onosInstall( options="-f",
135 node=ONOS4Ip )
136 onos5InstallResult = main.ONOSbench.onosInstall( options="-f",
137 node=ONOS5Ip )
138 onos6InstallResult = main.ONOSbench.onosInstall( options="-f",
139 node=ONOS6Ip )
140 onos7InstallResult = main.ONOSbench.onosInstall( options="-f",
141 node=ONOS7Ip )
142 onosInstallResult = onos1InstallResult and onos2InstallResult\
143 and onos3InstallResult and onos4InstallResult\
144 and onos5InstallResult and onos6InstallResult\
145 and onos7InstallResult
Jon Hallb1290e82014-11-18 16:17:48 -0500146
Jon Hall6aec96b2015-01-19 14:49:31 -0800147 main.step( "Checking if ONOS is up yet" )
148 # TODO check bundle:list?
149 for i in range( 2 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800150 onos1Isup = main.ONOSbench.isup( ONOS1Ip )
151 if not onos1Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800152 main.log.report( "ONOS1 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800153 main.ONOSbench.onosStop( ONOS1Ip )
154 main.ONOSbench.onosStart( ONOS1Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800155 onos2Isup = main.ONOSbench.isup( ONOS2Ip )
156 if not onos2Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800157 main.log.report( "ONOS2 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800158 main.ONOSbench.onosStop( ONOS2Ip )
159 main.ONOSbench.onosStart( ONOS2Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800160 onos3Isup = main.ONOSbench.isup( ONOS3Ip )
161 if not onos3Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800162 main.log.report( "ONOS3 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800163 main.ONOSbench.onosStop( ONOS3Ip )
164 main.ONOSbench.onosStart( ONOS3Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800165 onos4Isup = main.ONOSbench.isup( ONOS4Ip )
166 if not onos4Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800167 main.log.report( "ONOS4 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800168 main.ONOSbench.onosStop( ONOS4Ip )
169 main.ONOSbench.onosStart( ONOS4Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800170 onos5Isup = main.ONOSbench.isup( ONOS5Ip )
171 if not onos5Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800172 main.log.report( "ONOS5 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800173 main.ONOSbench.onosStop( ONOS5Ip )
174 main.ONOSbench.onosStart( ONOS5Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800175 onos6Isup = main.ONOSbench.isup( ONOS6Ip )
176 if not onos6Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800177 main.log.report( "ONOS6 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800178 main.ONOSbench.onosStop( ONOS6Ip )
179 main.ONOSbench.onosStart( ONOS6Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800180 onos7Isup = main.ONOSbench.isup( ONOS7Ip )
181 if not onos7Isup:
Jon Hall6aec96b2015-01-19 14:49:31 -0800182 main.log.report( "ONOS7 didn't start!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800183 main.ONOSbench.onosStop( ONOS7Ip )
184 main.ONOSbench.onosStart( ONOS7Ip )
Jon Hall8f89dda2015-01-22 16:03:33 -0800185 onosIsupResult = onos1Isup and onos2Isup and onos3Isup\
186 and onos4Isup and onos5Isup and onos6Isup and onos7Isup
187 if onosIsupResult == main.TRUE:
Jon Hallffb386d2014-11-21 13:43:38 -0800188 break
Jon Hallb1290e82014-11-18 16:17:48 -0500189
Jon Hall8f89dda2015-01-22 16:03:33 -0800190 cliResult1 = main.ONOScli1.startOnosCli( ONOS1Ip )
191 cliResult2 = main.ONOScli2.startOnosCli( ONOS2Ip )
192 cliResult3 = main.ONOScli3.startOnosCli( ONOS3Ip )
193 cliResult4 = main.ONOScli4.startOnosCli( ONOS4Ip )
194 cliResult5 = main.ONOScli5.startOnosCli( ONOS5Ip )
195 cliResult6 = main.ONOScli6.startOnosCli( ONOS6Ip )
196 cliResult7 = main.ONOScli7.startOnosCli( ONOS7Ip )
197 cliResults = cliResult1 and cliResult2 and cliResult3 and\
198 cliResult4 and cliResult5 and cliResult6 and cliResult7
Jon Hallb1290e82014-11-18 16:17:48 -0500199
Jon Hall6aec96b2015-01-19 14:49:31 -0800200 main.step( "Start Packet Capture MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800201 main.Mininet2.startTcpdump(
Jon Hall6aec96b2015-01-19 14:49:31 -0800202 str( main.params[ 'MNtcpdump' ][ 'folder' ] ) + str( main.TEST )
203 + "-MN.pcap",
204 intf=main.params[ 'MNtcpdump' ][ 'intf' ],
205 port=main.params[ 'MNtcpdump' ][ 'port' ] )
Jon Hallb1290e82014-11-18 16:17:48 -0500206
Jon Hall8f89dda2015-01-22 16:03:33 -0800207 case1Result = ( cleanInstallResult and packageResult and
208 cellResult and verifyResult and onosInstallResult
209 and onosIsupResult and cliResults )
Jon Hallb1290e82014-11-18 16:17:48 -0500210
Jon Hall8f89dda2015-01-22 16:03:33 -0800211 utilities.assert_equals( expect=main.TRUE, actual=case1Result,
212 onpass="Test startup successful",
213 onfail="Test startup NOT successful" )
Jon Hallb1290e82014-11-18 16:17:48 -0500214
Jon Hall8f89dda2015-01-22 16:03:33 -0800215 if case1Result == main.FALSE:
Jon Hallffb386d2014-11-21 13:43:38 -0800216 main.cleanup()
217 main.exit()
Jon Hallb1290e82014-11-18 16:17:48 -0500218
Jon Hall6aec96b2015-01-19 14:49:31 -0800219 def CASE2( self, main ):
220 """
Jon Hallb1290e82014-11-18 16:17:48 -0500221 Assign mastership to controllers
Jon Hall6aec96b2015-01-19 14:49:31 -0800222 """
Jon Hallb1290e82014-11-18 16:17:48 -0500223 import re
224
Jon Hall6aec96b2015-01-19 14:49:31 -0800225 main.log.report( "Assigning switches to controllers" )
226 main.case( "Assigning Controllers" )
227 main.step( "Assign switches to controllers" )
Jon Hallb1290e82014-11-18 16:17:48 -0500228
Jon Hall6aec96b2015-01-19 14:49:31 -0800229 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800230 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -0800231 sw=str( i ),
Jon Hall8f89dda2015-01-22 16:03:33 -0800232 count=numControllers,
233 ip1=ONOS1Ip, port1=ONOS1Port,
234 ip2=ONOS2Ip, port2=ONOS2Port,
235 ip3=ONOS3Ip, port3=ONOS3Port,
236 ip4=ONOS4Ip, port4=ONOS4Port,
237 ip5=ONOS5Ip, port5=ONOS5Port,
238 ip6=ONOS6Ip, port6=ONOS6Port,
239 ip7=ONOS7Ip, port7=ONOS7Port )
Jon Hallb1290e82014-11-18 16:17:48 -0500240
Jon Hall8f89dda2015-01-22 16:03:33 -0800241 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800242 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800243 response = main.Mininet1.getSwController( "s" + str( i ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800244 try:
Jon Hall6aec96b2015-01-19 14:49:31 -0800245 main.log.info( str( response ) )
Jon Hallffb386d2014-11-21 13:43:38 -0800246 except:
Jon Hall6aec96b2015-01-19 14:49:31 -0800247 main.log.info( repr( response ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800248 if re.search( "tcp:" + ONOS1Ip, response )\
249 and re.search( "tcp:" + ONOS2Ip, response )\
250 and re.search( "tcp:" + ONOS3Ip, response )\
251 and re.search( "tcp:" + ONOS4Ip, response )\
252 and re.search( "tcp:" + ONOS5Ip, response )\
253 and re.search( "tcp:" + ONOS6Ip, response )\
254 and re.search( "tcp:" + ONOS7Ip, response ):
255 mastershipCheck = mastershipCheck and main.TRUE
Jon Hallb1290e82014-11-18 16:17:48 -0500256 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800257 mastershipCheck = main.FALSE
258 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800259 main.log.report( "Switch mastership assigned correctly" )
260 utilities.assert_equals(
261 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800262 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800263 onpass="Switch mastership assigned correctly",
264 onfail="Switches not assigned correctly to controllers" )
Jon Hallb1290e82014-11-18 16:17:48 -0500265
Jon Hall6aec96b2015-01-19 14:49:31 -0800266 # Manually assign mastership to the controller we want
Jon Hall8f89dda2015-01-22 16:03:33 -0800267 roleCall = main.TRUE
268 roleCheck = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800269
Jon Hall6aec96b2015-01-19 14:49:31 -0800270 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800271 deviceId = main.ONOScli1.getDevice( "1000" ).get( 'id' )
272 roleCall = roleCall and main.ONOScli1.deviceRole(
273 deviceId,
274 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800275 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800276 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
277 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800278 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800279 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800280
Jon Hall6aec96b2015-01-19 14:49:31 -0800281 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800282 deviceId = main.ONOScli1.getDevice( "2800" ).get( 'id' )
283 roleCall = roleCall and main.ONOScli1.deviceRole(
284 deviceId,
285 ONOS1Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800286 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800287 if ONOS1Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
288 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800289 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800290 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800291
Jon Hall6aec96b2015-01-19 14:49:31 -0800292 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800293 deviceId = main.ONOScli1.getDevice( "2000" ).get( 'id' )
294 roleCall = roleCall and main.ONOScli1.deviceRole(
295 deviceId,
296 ONOS2Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800297 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800298 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
299 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800300 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800301 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800302
Jon Hall6aec96b2015-01-19 14:49:31 -0800303 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800304 deviceId = main.ONOScli1.getDevice( "3000" ).get( 'id' )
305 roleCall = roleCall and main.ONOScli1.deviceRole(
306 deviceId,
307 ONOS2Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800308 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800309 if ONOS2Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
310 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800311 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800312 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800313
Jon Hall6aec96b2015-01-19 14:49:31 -0800314 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800315 deviceId = main.ONOScli1.getDevice( "5000" ).get( 'id' )
316 roleCall = roleCall and main.ONOScli1.deviceRole(
317 deviceId,
318 ONOS3Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800319 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800320 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
321 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800322 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800323 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800324
Jon Hall6aec96b2015-01-19 14:49:31 -0800325 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800326 deviceId = main.ONOScli1.getDevice( "6000" ).get( 'id' )
327 roleCall = roleCall and main.ONOScli1.deviceRole(
328 deviceId,
329 ONOS3Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800330 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800331 if ONOS3Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
332 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800333 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800334 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800335
Jon Hall6aec96b2015-01-19 14:49:31 -0800336 # Assign switch
Jon Hall8f89dda2015-01-22 16:03:33 -0800337 deviceId = main.ONOScli1.getDevice( "3004" ).get( 'id' )
338 roleCall = roleCall and main.ONOScli1.deviceRole(
339 deviceId,
340 ONOS4Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800341 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800342 if ONOS4Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
343 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800344 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800345 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800346
Jon Hall6aec96b2015-01-19 14:49:31 -0800347 for i in range( 8, 18 ):
348 dpid = '3' + str( i ).zfill( 3 )
Jon Hall8f89dda2015-01-22 16:03:33 -0800349 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
350 roleCall = roleCall and main.ONOScli1.deviceRole(
351 deviceId,
352 ONOS5Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800353 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800354 if ONOS5Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
355 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800356 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800357 roleCheck = roleCheck and main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800358
Jon Hall8f89dda2015-01-22 16:03:33 -0800359 deviceId = main.ONOScli1.getDevice( "6007" ).get( 'id' )
360 roleCall = roleCall and main.ONOScli1.deviceRole(
361 deviceId,
362 ONOS6Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800363 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800364 if ONOS6Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
365 roleCheck = roleCheck and main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800366 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800367 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800368
Jon Hall6aec96b2015-01-19 14:49:31 -0800369 for i in range( 18, 28 ):
370 dpid = '6' + str( i ).zfill( 3 )
Jon Hall8f89dda2015-01-22 16:03:33 -0800371 deviceId = main.ONOScli1.getDevice( dpid ).get( 'id' )
372 roleCall = roleCall and main.ONOScli1.deviceRole(
373 deviceId,
374 ONOS7Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -0800375 # Check assignment
Jon Hall8f89dda2015-01-22 16:03:33 -0800376 if ONOS7Ip in main.ONOScli1.getRole( deviceId ).get( 'master' ):
377 roleCheck = roleCheck and main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800378 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800379 roleCheck = roleCheck and main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800380
Jon Hall6aec96b2015-01-19 14:49:31 -0800381 utilities.assert_equals(
382 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800383 actual=roleCall,
Jon Hall6aec96b2015-01-19 14:49:31 -0800384 onpass="Re-assigned switch mastership to designated controller",
Jon Hall8f89dda2015-01-22 16:03:33 -0800385 onfail="Something wrong with deviceRole calls" )
Jon Hall94fd0472014-12-08 11:52:42 -0800386
Jon Hall6aec96b2015-01-19 14:49:31 -0800387 utilities.assert_equals(
388 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800389 actual=roleCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800390 onpass="Switches were successfully reassigned to designated " +
391 "controller",
392 onfail="Switches were not successfully reassigned" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800393 mastershipCheck = mastershipCheck and roleCall and roleCheck
394 utilities.assert_equals( expect=main.TRUE, actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800395 onpass="Switch mastership correctly assigned",
396 onfail="Error in (re)assigning switch" +
Jon Hall8f89dda2015-01-22 16:03:33 -0800397 " mastership" )
Jon Hallb1290e82014-11-18 16:17:48 -0500398
Jon Hall6aec96b2015-01-19 14:49:31 -0800399 def CASE3( self, main ):
Jon Hallb1290e82014-11-18 16:17:48 -0500400 """
401 Assign intents
Jon Hallb1290e82014-11-18 16:17:48 -0500402 """
403 import time
Jon Hall6aec96b2015-01-19 14:49:31 -0800404 main.log.report( "Adding host intents" )
405 main.case( "Adding host Intents" )
Jon Hallb1290e82014-11-18 16:17:48 -0500406
Jon Hall8f89dda2015-01-22 16:03:33 -0800407 main.step( "Discovering Hosts( Via pingall for now )" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800408 # FIXME: Once we have a host discovery mechanism, use that instead
Jon Hallb1290e82014-11-18 16:17:48 -0500409
Jon Hall6aec96b2015-01-19 14:49:31 -0800410 # install onos-app-fwd
411 main.log.info( "Install reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800412 main.ONOScli1.featureInstall( "onos-app-fwd" )
413 main.ONOScli2.featureInstall( "onos-app-fwd" )
414 main.ONOScli3.featureInstall( "onos-app-fwd" )
415 main.ONOScli4.featureInstall( "onos-app-fwd" )
416 main.ONOScli5.featureInstall( "onos-app-fwd" )
417 main.ONOScli6.featureInstall( "onos-app-fwd" )
418 main.ONOScli7.featureInstall( "onos-app-fwd" )
Jon Hall94fd0472014-12-08 11:52:42 -0800419
Jon Hall6aec96b2015-01-19 14:49:31 -0800420 # REACTIVE FWD test
Jon Hall8f89dda2015-01-22 16:03:33 -0800421 pingResult = main.FALSE
Jon Hallb1290e82014-11-18 16:17:48 -0500422 time1 = time.time()
Jon Hall8f89dda2015-01-22 16:03:33 -0800423 pingResult = main.Mininet1.pingall()
Jon Hall529a37f2015-01-28 10:02:00 -0800424 utilities.assert_equals(
425 expect=main.TRUE,
426 actual=pingResult,
427 onpass="Reactive Pingall test passed",
428 onfail="Reactive Pingall failed, one or more ping pairs failed" )
Jon Hallb1290e82014-11-18 16:17:48 -0500429 time2 = time.time()
Jon Hall6aec96b2015-01-19 14:49:31 -0800430 main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
Jon Hallb1290e82014-11-18 16:17:48 -0500431
Jon Hall6aec96b2015-01-19 14:49:31 -0800432 # uninstall onos-app-fwd
433 main.log.info( "Uninstall reactive forwarding app" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800434 main.ONOScli1.featureUninstall( "onos-app-fwd" )
435 main.ONOScli2.featureUninstall( "onos-app-fwd" )
436 main.ONOScli3.featureUninstall( "onos-app-fwd" )
437 main.ONOScli4.featureUninstall( "onos-app-fwd" )
438 main.ONOScli5.featureUninstall( "onos-app-fwd" )
439 main.ONOScli6.featureUninstall( "onos-app-fwd" )
440 main.ONOScli7.featureUninstall( "onos-app-fwd" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800441 # timeout for fwd flows
442 time.sleep( 10 )
Jon Hallb1290e82014-11-18 16:17:48 -0500443
Jon Hall6aec96b2015-01-19 14:49:31 -0800444 main.step( "Add host intents" )
445 # TODO: move the host numbers to params
Jon Hall8f89dda2015-01-22 16:03:33 -0800446 intentAddResult = True
Jon Hall6aec96b2015-01-19 14:49:31 -0800447 for i in range( 8, 18 ):
448 main.log.info( "Adding host intent between h" + str( i ) +
449 " and h" + str( i + 10 ) )
450 host1 = "00:00:00:00:00:" + \
451 str( hex( i )[ 2: ] ).zfill( 2 ).upper()
452 host2 = "00:00:00:00:00:" + \
453 str( hex( i + 10 )[ 2: ] ).zfill( 2 ).upper()
Jon Hall1b8f54a2015-02-04 13:24:20 -0800454 # NOTE: getHost can return None
455 host1Dict = main.ONOScli1.getHost( host1 )
456 host2Dict = main.ONOScli1.getHost( host2 )
457 host1Id = None
458 host2Id = None
459 if host1Dict and host2Dict:
460 host1Id = host1Dict.get( 'id', None )
461 host2Id = host2Dict.get( 'id', None )
Jon Hall8f89dda2015-01-22 16:03:33 -0800462 if host1Id and host2Id:
Jon Hall21270ac2015-02-16 17:59:55 -0800463 # TODO: distribute the intents across onos nodes
Jon Hall8f89dda2015-01-22 16:03:33 -0800464 tmpResult = main.ONOScli1.addHostIntent(
465 host1Id,
466 host2Id )
Jon Hall669173b2014-12-17 11:36:30 -0800467 else:
Jon Hall8f89dda2015-01-22 16:03:33 -0800468 main.log.error( "Error, getHost() failed" )
Jon Hall1b8f54a2015-02-04 13:24:20 -0800469 main.log.warn( json.dumps( json.loads( main.ONOScli1.hosts() ),
470 sort_keys=True,
471 indent=4,
472 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800473 tmpResult = main.FALSE
474 intentAddResult = bool( pingResult and intentAddResult
475 and tmpResult )
476 # TODO Check that intents were added?
Jon Hall1b8f54a2015-02-04 13:24:20 -0800477 # Print the intent states
478 intents = main.ONOScli1.intents( )
479 intentStates = []
480 for intent in json.loads( intents ): # Iter through intents of a node
481 intentStates.append( intent.get( 'state', None ) )
482 out = [ (i, intentStates.count( i ) ) for i in set( intentStates ) ]
483 main.log.info( dict( out ) )
484
Jon Hall6aec96b2015-01-19 14:49:31 -0800485 utilities.assert_equals(
486 expect=True,
Jon Hall8f89dda2015-01-22 16:03:33 -0800487 actual=intentAddResult,
488 onpass="Pushed host intents to ONOS",
489 onfail="Error in pushing host intents to ONOS" )
Jon Hall6aec96b2015-01-19 14:49:31 -0800490 # TODO Check if intents all exist in datastore
Jon Hallb1290e82014-11-18 16:17:48 -0500491
Jon Hall6aec96b2015-01-19 14:49:31 -0800492 def CASE4( self, main ):
Jon Hallb1290e82014-11-18 16:17:48 -0500493 """
494 Ping across added host intents
495 """
Jon Hall368769f2014-11-19 15:43:35 -0800496 description = " Ping across added host intents"
Jon Hall6aec96b2015-01-19 14:49:31 -0800497 main.log.report( description )
498 main.case( description )
Jon Hall8f89dda2015-01-22 16:03:33 -0800499 PingResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800500 for i in range( 8, 18 ):
501 ping = main.Mininet1.pingHost(
502 src="h" + str( i ), target="h" + str( i + 10 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800503 PingResult = PingResult and ping
Jon Hall6aec96b2015-01-19 14:49:31 -0800504 if ping == main.FALSE:
505 main.log.warn( "Ping failed between h" + str( i ) +
506 " and h" + str( i + 10 ) )
507 elif ping == main.TRUE:
508 main.log.info( "Ping test passed!" )
Jon Hall21270ac2015-02-16 17:59:55 -0800509 # Don't set PingResult or you'd override failures
Jon Hall8f89dda2015-01-22 16:03:33 -0800510 if PingResult == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800511 main.log.report(
512 "Intents have not been installed correctly, pings failed." )
Jon Hall529a37f2015-01-28 10:02:00 -0800513 #TODO: pretty print
514 main.log.warn( "ONSO1 intents: " )
515 main.log.warn( json.dumps( json.loads( main.ONOScli1.intents() ),
516 sort_keys=True,
517 indent=4,
518 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800519 if PingResult == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -0800520 main.log.report(
521 "Intents have been installed correctly and verified by pings" )
522 utilities.assert_equals(
523 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800524 actual=PingResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800525 onpass="Intents have been installed correctly and pings work",
526 onfail="Intents have not been installed correctly, pings failed." )
Jon Hallb1290e82014-11-18 16:17:48 -0500527
Jon Hall6aec96b2015-01-19 14:49:31 -0800528 def CASE5( self, main ):
529 """
Jon Hallb1290e82014-11-18 16:17:48 -0500530 Reading state of ONOS
Jon Hall6aec96b2015-01-19 14:49:31 -0800531 """
Jon Hallb1290e82014-11-18 16:17:48 -0500532 import json
Jon Hall6aec96b2015-01-19 14:49:31 -0800533 # assumes that sts is already in you PYTHONPATH
534 from sts.topology.teston_topology import TestONTopology
Jon Hallb1290e82014-11-18 16:17:48 -0500535
Jon Hall6aec96b2015-01-19 14:49:31 -0800536 main.log.report( "Setting up and gathering data for current state" )
537 main.case( "Setting up and gathering data for current state" )
538 # The general idea for this test case is to pull the state of
539 # ( intents,flows, topology,... ) from each ONOS node
540 # We can then compare them with eachother and also with past states
Jon Hallb1290e82014-11-18 16:17:48 -0500541
Jon Hall6aec96b2015-01-19 14:49:31 -0800542 main.step( "Get the Mastership of each switch from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800543 global mastershipState
544 mastershipState = []
Jon Hall94fd0472014-12-08 11:52:42 -0800545
Jon Hall6aec96b2015-01-19 14:49:31 -0800546 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -0800547 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
548 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
549 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
550 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
551 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
552 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
553 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
554 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
555 ONOS3MasterNotNull and ONOS4MasterNotNull and\
556 ONOS5MasterNotNull and ONOS6MasterNotNull and\
557 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -0800558 utilities.assert_equals(
559 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800560 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -0800561 onpass="Each device has a master",
562 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -0800563
Jon Hall8f89dda2015-01-22 16:03:33 -0800564 ONOS1Mastership = main.ONOScli1.roles()
565 ONOS2Mastership = main.ONOScli2.roles()
566 ONOS3Mastership = main.ONOScli3.roles()
567 ONOS4Mastership = main.ONOScli4.roles()
568 ONOS5Mastership = main.ONOScli5.roles()
569 ONOS6Mastership = main.ONOScli6.roles()
570 ONOS7Mastership = main.ONOScli7.roles()
Jon Hall8f89dda2015-01-22 16:03:33 -0800571 if "Error" in ONOS1Mastership or not ONOS1Mastership\
572 or "Error" in ONOS2Mastership or not ONOS2Mastership\
573 or "Error" in ONOS3Mastership or not ONOS3Mastership\
574 or "Error" in ONOS4Mastership or not ONOS4Mastership\
575 or "Error" in ONOS5Mastership or not ONOS5Mastership\
576 or "Error" in ONOS6Mastership or not ONOS6Mastership\
577 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -0800578 main.log.report( "Error in getting ONOS roles" )
579 main.log.warn(
580 "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800581 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800582 main.log.warn(
583 "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800584 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800585 main.log.warn(
586 "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800587 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800588 main.log.warn(
589 "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800590 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800591 main.log.warn(
592 "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800593 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800594 main.log.warn(
595 "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800596 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -0800597 main.log.warn(
598 "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800599 repr( ONOS7Mastership ) )
600 consistentMastership = main.FALSE
601 elif ONOS1Mastership == ONOS2Mastership\
602 and ONOS1Mastership == ONOS3Mastership\
603 and ONOS1Mastership == ONOS4Mastership\
604 and ONOS1Mastership == ONOS5Mastership\
605 and ONOS1Mastership == ONOS6Mastership\
606 and ONOS1Mastership == ONOS7Mastership:
607 mastershipState = ONOS1Mastership
608 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800609 main.log.report(
610 "Switch roles are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -0500611 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800612 main.log.warn(
613 "ONOS1 roles: ",
614 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800615 json.loads( ONOS1Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800616 sort_keys=True,
617 indent=4,
618 separators=(
619 ',',
620 ': ' ) ) )
621 main.log.warn(
622 "ONOS2 roles: ",
623 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800624 json.loads( ONOS2Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800625 sort_keys=True,
626 indent=4,
627 separators=(
628 ',',
629 ': ' ) ) )
630 main.log.warn(
631 "ONOS3 roles: ",
632 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800633 json.loads( ONOS3Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800634 sort_keys=True,
635 indent=4,
636 separators=(
637 ',',
638 ': ' ) ) )
639 main.log.warn(
640 "ONOS4 roles: ",
641 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800642 json.loads( ONOS4Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800643 sort_keys=True,
644 indent=4,
645 separators=(
646 ',',
647 ': ' ) ) )
648 main.log.warn(
649 "ONOS5 roles: ",
650 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800651 json.loads( ONOS5Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800652 sort_keys=True,
653 indent=4,
654 separators=(
655 ',',
656 ': ' ) ) )
657 main.log.warn(
658 "ONOS6 roles: ",
659 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800660 json.loads( ONOS6Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800661 sort_keys=True,
662 indent=4,
663 separators=(
664 ',',
665 ': ' ) ) )
666 main.log.warn(
667 "ONOS7 roles: ",
668 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800669 json.loads( ONOS7Mastership ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800670 sort_keys=True,
671 indent=4,
672 separators=(
673 ',',
674 ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800675 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -0800676 utilities.assert_equals(
677 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800678 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -0800679 onpass="Switch roles are consistent across all ONOS nodes",
680 onfail="ONOS nodes have different views of switch roles" )
Jon Hallb1290e82014-11-18 16:17:48 -0500681
Jon Hall6aec96b2015-01-19 14:49:31 -0800682 main.step( "Get the intents from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800683 global intentState
684 intentState = []
685 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
686 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
687 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
688 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
689 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
690 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
691 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
692 intentCheck = main.FALSE
693 if "Error" in ONOS1Intents or not ONOS1Intents\
694 or "Error" in ONOS2Intents or not ONOS2Intents\
695 or "Error" in ONOS3Intents or not ONOS3Intents\
696 or "Error" in ONOS4Intents or not ONOS4Intents\
697 or "Error" in ONOS5Intents or not ONOS5Intents\
698 or "Error" in ONOS6Intents or not ONOS6Intents\
699 or "Error" in ONOS7Intents or not ONOS7Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -0800700 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800701 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
702 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
703 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
704 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
705 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
706 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
707 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
708 elif ONOS1Intents == ONOS2Intents\
709 and ONOS1Intents == ONOS3Intents\
710 and ONOS1Intents == ONOS4Intents\
711 and ONOS1Intents == ONOS5Intents\
712 and ONOS1Intents == ONOS6Intents\
713 and ONOS1Intents == ONOS7Intents:
714 intentState = ONOS1Intents
715 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800716 main.log.report( "Intents are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -0500717 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800718 main.log.warn(
719 "ONOS1 intents: ",
720 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800721 json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800722 sort_keys=True,
723 indent=4,
724 separators=(
725 ',',
726 ': ' ) ) )
727 main.log.warn(
728 "ONOS2 intents: ",
729 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800730 json.loads( ONOS2Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800731 sort_keys=True,
732 indent=4,
733 separators=(
734 ',',
735 ': ' ) ) )
736 main.log.warn(
737 "ONOS3 intents: ",
738 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800739 json.loads( ONOS3Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800740 sort_keys=True,
741 indent=4,
742 separators=(
743 ',',
744 ': ' ) ) )
745 main.log.warn(
746 "ONOS4 intents: ",
747 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800748 json.loads( ONOS4Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800749 sort_keys=True,
750 indent=4,
751 separators=(
752 ',',
753 ': ' ) ) )
754 main.log.warn(
755 "ONOS5 intents: ",
756 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800757 json.loads( ONOS5Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800758 sort_keys=True,
759 indent=4,
760 separators=(
761 ',',
762 ': ' ) ) )
763 main.log.warn(
764 "ONOS6 intents: ",
765 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800766 json.loads( ONOS6Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800767 sort_keys=True,
768 indent=4,
769 separators=(
770 ',',
771 ': ' ) ) )
772 main.log.warn(
773 "ONOS7 intents: ",
774 json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -0800775 json.loads( ONOS7Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -0800776 sort_keys=True,
777 indent=4,
778 separators=(
779 ',',
780 ': ' ) ) )
781 utilities.assert_equals(
782 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800783 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800784 onpass="Intents are consistent across all ONOS nodes",
785 onfail="ONOS nodes have different views of intents" )
Jon Hallb1290e82014-11-18 16:17:48 -0500786
Jon Hall6aec96b2015-01-19 14:49:31 -0800787 main.step( "Get the flows from each controller" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800788 global flowState
789 flowState = []
790 ONOS1Flows = main.ONOScli1.flows( jsonFormat=True )
791 ONOS2Flows = main.ONOScli2.flows( jsonFormat=True )
792 ONOS3Flows = main.ONOScli3.flows( jsonFormat=True )
793 ONOS4Flows = main.ONOScli4.flows( jsonFormat=True )
794 ONOS5Flows = main.ONOScli5.flows( jsonFormat=True )
795 ONOS6Flows = main.ONOScli6.flows( jsonFormat=True )
796 ONOS7Flows = main.ONOScli7.flows( jsonFormat=True )
797 ONOS1FlowsJson = json.loads( ONOS1Flows )
798 ONOS2FlowsJson = json.loads( ONOS2Flows )
799 ONOS3FlowsJson = json.loads( ONOS3Flows )
800 ONOS4FlowsJson = json.loads( ONOS4Flows )
801 ONOS5FlowsJson = json.loads( ONOS5Flows )
802 ONOS6FlowsJson = json.loads( ONOS6Flows )
803 ONOS7FlowsJson = json.loads( ONOS7Flows )
804 flowCheck = main.FALSE
805 if "Error" in ONOS1Flows or not ONOS1Flows\
806 or "Error" in ONOS2Flows or not ONOS2Flows\
807 or "Error" in ONOS3Flows or not ONOS3Flows\
808 or "Error" in ONOS4Flows or not ONOS4Flows\
809 or "Error" in ONOS5Flows or not ONOS5Flows\
810 or "Error" in ONOS6Flows or not ONOS6Flows\
811 or "Error" in ONOS7Flows or not ONOS7Flows:
Jon Hall6aec96b2015-01-19 14:49:31 -0800812 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -0800813 main.log.warn( "ONOS1 flows repsponse: " + ONOS1Flows )
814 main.log.warn( "ONOS2 flows repsponse: " + ONOS2Flows )
815 main.log.warn( "ONOS3 flows repsponse: " + ONOS3Flows )
816 main.log.warn( "ONOS4 flows repsponse: " + ONOS4Flows )
817 main.log.warn( "ONOS5 flows repsponse: " + ONOS5Flows )
818 main.log.warn( "ONOS6 flows repsponse: " + ONOS6Flows )
819 main.log.warn( "ONOS7 flows repsponse: " + ONOS7Flows )
820 elif len( ONOS1FlowsJson ) == len( ONOS2FlowsJson )\
821 and len( ONOS1FlowsJson ) == len( ONOS3FlowsJson )\
822 and len( ONOS1FlowsJson ) == len( ONOS4FlowsJson )\
823 and len( ONOS1FlowsJson ) == len( ONOS5FlowsJson )\
824 and len( ONOS1FlowsJson ) == len( ONOS6FlowsJson )\
825 and len( ONOS1FlowsJson ) == len( ONOS7FlowsJson ):
Jon Hall6aec96b2015-01-19 14:49:31 -0800826 # TODO: Do a better check, maybe compare flows on switches?
Jon Hall8f89dda2015-01-22 16:03:33 -0800827 flowState = ONOS1Flows
828 flowCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -0800829 main.log.report( "Flow count is consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -0500830 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800831 main.log.warn( "ONOS1 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800832 json.dumps( ONOS1FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800833 indent=4, separators=( ',', ': ' ) ) )
834 main.log.warn( "ONOS2 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800835 json.dumps( ONOS2FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800836 indent=4, separators=( ',', ': ' ) ) )
837 main.log.warn( "ONOS3 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800838 json.dumps( ONOS3FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800839 indent=4, separators=( ',', ': ' ) ) )
840 main.log.warn( "ONOS4 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800841 json.dumps( ONOS4FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800842 indent=4, separators=( ',', ': ' ) ) )
843 main.log.warn( "ONOS5 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800844 json.dumps( ONOS5FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800845 indent=4, separators=( ',', ': ' ) ) )
846 main.log.warn( "ONOS6 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800847 json.dumps( ONOS6FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800848 indent=4, separators=( ',', ': ' ) ) )
849 main.log.warn( "ONOS7 flows: " +
Jon Hall8f89dda2015-01-22 16:03:33 -0800850 json.dumps( ONOS7FlowsJson, sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -0800851 indent=4, separators=( ',', ': ' ) ) )
852 utilities.assert_equals(
853 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800854 actual=flowCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -0800855 onpass="The flow count is consistent across all ONOS nodes",
856 onfail="ONOS nodes have different flow counts" )
Jon Hallb1290e82014-11-18 16:17:48 -0500857
Jon Hall6aec96b2015-01-19 14:49:31 -0800858 main.step( "Get the OF Table entries" )
Jon Hallb1290e82014-11-18 16:17:48 -0500859 global flows
Jon Hall6aec96b2015-01-19 14:49:31 -0800860 flows = []
861 for i in range( 1, 29 ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800862 flows.append( main.Mininet2.getFlowTable( 1.3, "s" + str( i ) ) )
Jon Hallb1290e82014-11-18 16:17:48 -0500863
Jon Hall6aec96b2015-01-19 14:49:31 -0800864 # TODO: Compare switch flow tables with ONOS flow tables
Jon Hallb1290e82014-11-18 16:17:48 -0500865
Jon Hall6aec96b2015-01-19 14:49:31 -0800866 main.step( "Start continuous pings" )
867 main.Mininet2.pingLong(
868 src=main.params[ 'PING' ][ 'source1' ],
869 target=main.params[ 'PING' ][ 'target1' ],
870 pingTime=500 )
871 main.Mininet2.pingLong(
872 src=main.params[ 'PING' ][ 'source2' ],
873 target=main.params[ 'PING' ][ 'target2' ],
874 pingTime=500 )
875 main.Mininet2.pingLong(
876 src=main.params[ 'PING' ][ 'source3' ],
877 target=main.params[ 'PING' ][ 'target3' ],
878 pingTime=500 )
879 main.Mininet2.pingLong(
880 src=main.params[ 'PING' ][ 'source4' ],
881 target=main.params[ 'PING' ][ 'target4' ],
882 pingTime=500 )
883 main.Mininet2.pingLong(
884 src=main.params[ 'PING' ][ 'source5' ],
885 target=main.params[ 'PING' ][ 'target5' ],
886 pingTime=500 )
887 main.Mininet2.pingLong(
888 src=main.params[ 'PING' ][ 'source6' ],
889 target=main.params[ 'PING' ][ 'target6' ],
890 pingTime=500 )
891 main.Mininet2.pingLong(
892 src=main.params[ 'PING' ][ 'source7' ],
893 target=main.params[ 'PING' ][ 'target7' ],
894 pingTime=500 )
895 main.Mininet2.pingLong(
896 src=main.params[ 'PING' ][ 'source8' ],
897 target=main.params[ 'PING' ][ 'target8' ],
898 pingTime=500 )
899 main.Mininet2.pingLong(
900 src=main.params[ 'PING' ][ 'source9' ],
901 target=main.params[ 'PING' ][ 'target9' ],
902 pingTime=500 )
903 main.Mininet2.pingLong(
904 src=main.params[ 'PING' ][ 'source10' ],
905 target=main.params[ 'PING' ][ 'target10' ],
906 pingTime=500 )
Jon Hallb1290e82014-11-18 16:17:48 -0500907
Jon Hall6aec96b2015-01-19 14:49:31 -0800908 main.step( "Create TestONTopology object" )
Jon Hallb1290e82014-11-18 16:17:48 -0500909 ctrls = []
910 count = 1
911 while True:
912 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -0800913 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
914 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
915 temp = temp + ( "ONOS" + str( count ), )
916 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
917 temp = temp + \
918 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
919 ctrls.append( temp )
Jon Hallb1290e82014-11-18 16:17:48 -0500920 count = count + 1
921 else:
922 break
Jon Hall6aec96b2015-01-19 14:49:31 -0800923 MNTopo = TestONTopology(
924 main.Mininet1,
925 ctrls ) # can also add Intent API info for intent operations
Jon Hallb1290e82014-11-18 16:17:48 -0500926
Jon Hall6aec96b2015-01-19 14:49:31 -0800927 main.step( "Collecting topology information from ONOS" )
Jon Hallb1290e82014-11-18 16:17:48 -0500928 devices = []
929 devices.append( main.ONOScli1.devices() )
930 devices.append( main.ONOScli2.devices() )
931 devices.append( main.ONOScli3.devices() )
932 devices.append( main.ONOScli4.devices() )
933 devices.append( main.ONOScli5.devices() )
934 devices.append( main.ONOScli6.devices() )
935 devices.append( main.ONOScli7.devices() )
Jon Hallb1290e82014-11-18 16:17:48 -0500936 hosts = []
937 hosts.append( main.ONOScli1.hosts() )
938 hosts.append( main.ONOScli2.hosts() )
939 hosts.append( main.ONOScli3.hosts() )
940 hosts.append( main.ONOScli4.hosts() )
941 hosts.append( main.ONOScli5.hosts() )
942 hosts.append( main.ONOScli6.hosts() )
943 hosts.append( main.ONOScli7.hosts() )
Jon Hallb1290e82014-11-18 16:17:48 -0500944 ports = []
945 ports.append( main.ONOScli1.ports() )
946 ports.append( main.ONOScli2.ports() )
947 ports.append( main.ONOScli3.ports() )
948 ports.append( main.ONOScli4.ports() )
949 ports.append( main.ONOScli5.ports() )
950 ports.append( main.ONOScli6.ports() )
951 ports.append( main.ONOScli7.ports() )
952 links = []
953 links.append( main.ONOScli1.links() )
954 links.append( main.ONOScli2.links() )
955 links.append( main.ONOScli3.links() )
956 links.append( main.ONOScli4.links() )
957 links.append( main.ONOScli5.links() )
958 links.append( main.ONOScli6.links() )
959 links.append( main.ONOScli7.links() )
Jon Hall94fd0472014-12-08 11:52:42 -0800960 clusters = []
961 clusters.append( main.ONOScli1.clusters() )
962 clusters.append( main.ONOScli2.clusters() )
963 clusters.append( main.ONOScli3.clusters() )
964 clusters.append( main.ONOScli4.clusters() )
965 clusters.append( main.ONOScli5.clusters() )
966 clusters.append( main.ONOScli6.clusters() )
967 clusters.append( main.ONOScli7.clusters() )
Jon Hall529a37f2015-01-28 10:02:00 -0800968 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -0800969
Jon Hall6aec96b2015-01-19 14:49:31 -0800970 # hosts
Jon Hall8f89dda2015-01-22 16:03:33 -0800971 consistentHostsResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800972 for controller in range( len( hosts ) ):
Jon Hall8f89dda2015-01-22 16:03:33 -0800973 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -0800974 if "Error" not in hosts[ controller ]:
975 if hosts[ controller ] == hosts[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -0800976 continue
Jon Hall6aec96b2015-01-19 14:49:31 -0800977 else: # hosts not consistent
978 main.log.report( "hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -0800979 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -0800980 " is inconsistent with ONOS1" )
981 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -0800982 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -0800983
984 else:
Jon Hall6aec96b2015-01-19 14:49:31 -0800985 main.log.report( "Error in getting ONOS hosts from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -0800986 controllerStr )
987 consistentHostsResult = main.FALSE
988 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -0800989 " hosts response: " +
990 repr( hosts[ controller ] ) )
991 utilities.assert_equals(
992 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -0800993 actual=consistentHostsResult,
Jon Hall6aec96b2015-01-19 14:49:31 -0800994 onpass="Hosts view is consistent across all ONOS nodes",
995 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -0800996
Jon Hall6aec96b2015-01-19 14:49:31 -0800997 # Strongly connected clusters of devices
Jon Hall8f89dda2015-01-22 16:03:33 -0800998 consistentClustersResult = main.TRUE
Jon Hall94fd0472014-12-08 11:52:42 -0800999 for controller in range( len( clusters ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001000 if "Error" not in clusters[ controller ]:
1001 if clusters[ controller ] == clusters[ 0 ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001002 continue
Jon Hall6aec96b2015-01-19 14:49:31 -08001003 else: # clusters not consistent
1004 main.log.report( "clusters from ONOS" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001005 controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001006 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001007 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001008
1009 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001010 main.log.report( "Error in getting dataplane clusters " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001011 "from ONOS" + controllerStr )
1012 consistentClustersResult = main.FALSE
1013 main.log.warn( "ONOS" + controllerStr +
Jon Hall6aec96b2015-01-19 14:49:31 -08001014 " clusters response: " +
1015 repr( clusters[ controller ] ) )
1016 utilities.assert_equals(
1017 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001018 actual=consistentClustersResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001019 onpass="Clusters view is consistent across all ONOS nodes",
1020 onfail="ONOS nodes have different views of clusters" )
1021 # there should always only be one cluster
Jon Hall8f89dda2015-01-22 16:03:33 -08001022 numClusters = len( json.loads( clusters[ 0 ] ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001023 utilities.assert_equals(
1024 expect=1,
Jon Hall8f89dda2015-01-22 16:03:33 -08001025 actual=numClusters,
Jon Hall6aec96b2015-01-19 14:49:31 -08001026 onpass="ONOS shows 1 SCC",
1027 onfail="ONOS shows " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001028 str( numClusters ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001029 " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001030
Jon Hall6aec96b2015-01-19 14:49:31 -08001031 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001032 devicesResults = main.TRUE
1033 portsResults = main.TRUE
1034 linksResults = main.TRUE
1035 for controller in range( numControllers ):
1036 controllerStr = str( controller + 1 )
Jon Hall6aec96b2015-01-19 14:49:31 -08001037 if devices[ controller ] or "Error" not in devices[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001038 currentDevicesResult = main.Mininet1.compareSwitches(
Jon Hall6aec96b2015-01-19 14:49:31 -08001039 MNTopo,
1040 json.loads(
1041 devices[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001042 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001043 currentDevicesResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001044 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001045 actual=currentDevicesResult,
1046 onpass="ONOS" + controllerStr +
1047 " Switches view is correct",
1048 onfail="ONOS" + controllerStr +
1049 " Switches view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001050
Jon Hall6aec96b2015-01-19 14:49:31 -08001051 if ports[ controller ] or "Error" not in ports[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001052 currentPortsResult = main.Mininet1.comparePorts(
Jon Hall6aec96b2015-01-19 14:49:31 -08001053 MNTopo,
1054 json.loads(
1055 ports[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001056 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001057 currentPortsResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001058 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001059 actual=currentPortsResult,
1060 onpass="ONOS" + controllerStr +
1061 " ports view is correct",
1062 onfail="ONOS" + controllerStr +
1063 " ports view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001064
Jon Hall6aec96b2015-01-19 14:49:31 -08001065 if links[ controller ] or "Error" not in links[ controller ]:
Jon Hall8f89dda2015-01-22 16:03:33 -08001066 currentLinksResult = main.Mininet1.compareLinks(
Jon Hall6aec96b2015-01-19 14:49:31 -08001067 MNTopo,
1068 json.loads(
1069 links[ controller ] ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001070 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08001071 currentLinksResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001072 utilities.assert_equals( expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001073 actual=currentLinksResult,
1074 onpass="ONOS" + controllerStr +
1075 " links view is correct",
1076 onfail="ONOS" + controllerStr +
1077 " links view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001078
Jon Hall8f89dda2015-01-22 16:03:33 -08001079 devicesResults = devicesResults and currentDevicesResult
1080 portsResults = portsResults and currentPortsResult
1081 linksResults = linksResults and currentLinksResult
Jon Hallb1290e82014-11-18 16:17:48 -05001082
Jon Hall8f89dda2015-01-22 16:03:33 -08001083 topoResult = devicesResults and portsResults and linksResults\
Jon Hall529a37f2015-01-28 10:02:00 -08001084 and consistentHostsResult and consistentClustersResult
Jon Hall8f89dda2015-01-22 16:03:33 -08001085 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1086 onpass="Topology Check Test successful",
1087 onfail="Topology Check Test NOT successful" )
Jon Hallb1290e82014-11-18 16:17:48 -05001088
Jon Hall8f89dda2015-01-22 16:03:33 -08001089 finalAssert = main.TRUE
1090 finalAssert = finalAssert and topoResult and flowCheck \
1091 and intentCheck and consistentMastership and rolesNotNull
1092 utilities.assert_equals( expect=main.TRUE, actual=finalAssert,
1093 onpass="State check successful",
1094 onfail="State check NOT successful" )
Jon Hallb1290e82014-11-18 16:17:48 -05001095
Jon Hall6aec96b2015-01-19 14:49:31 -08001096 def CASE6( self, main ):
1097 """
Jon Hallb1290e82014-11-18 16:17:48 -05001098 The Failure case. Since this is the Sanity test, we do nothing.
Jon Hall6aec96b2015-01-19 14:49:31 -08001099 """
Jon Hall368769f2014-11-19 15:43:35 -08001100 import time
Jon Hall6aec96b2015-01-19 14:49:31 -08001101 main.log.report( "Wait 60 seconds instead of inducing a failure" )
1102 time.sleep( 60 )
1103 utilities.assert_equals(
1104 expect=main.TRUE,
1105 actual=main.TRUE,
1106 onpass="Sleeping 60 seconds",
1107 onfail="Something is terribly wrong with my math" )
Jon Hallb1290e82014-11-18 16:17:48 -05001108
Jon Hall6aec96b2015-01-19 14:49:31 -08001109 def CASE7( self, main ):
1110 """
Jon Hall368769f2014-11-19 15:43:35 -08001111 Check state after ONOS failure
Jon Hall6aec96b2015-01-19 14:49:31 -08001112 """
Jon Hallb1290e82014-11-18 16:17:48 -05001113 import json
Jon Hall6aec96b2015-01-19 14:49:31 -08001114 main.case( "Running ONOS Constant State Tests" )
Jon Hallb1290e82014-11-18 16:17:48 -05001115
Jon Hall6aec96b2015-01-19 14:49:31 -08001116 # Assert that each device has a master
Jon Hall8f89dda2015-01-22 16:03:33 -08001117 ONOS1MasterNotNull = main.ONOScli1.rolesNotNull()
1118 ONOS2MasterNotNull = main.ONOScli2.rolesNotNull()
1119 ONOS3MasterNotNull = main.ONOScli3.rolesNotNull()
1120 ONOS4MasterNotNull = main.ONOScli4.rolesNotNull()
1121 ONOS5MasterNotNull = main.ONOScli5.rolesNotNull()
1122 ONOS6MasterNotNull = main.ONOScli6.rolesNotNull()
1123 ONOS7MasterNotNull = main.ONOScli7.rolesNotNull()
1124 rolesNotNull = ONOS1MasterNotNull and ONOS2MasterNotNull and\
1125 ONOS3MasterNotNull and ONOS4MasterNotNull and\
1126 ONOS5MasterNotNull and ONOS6MasterNotNull and\
1127 ONOS7MasterNotNull
Jon Hall6aec96b2015-01-19 14:49:31 -08001128 utilities.assert_equals(
1129 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001130 actual=rolesNotNull,
Jon Hall6aec96b2015-01-19 14:49:31 -08001131 onpass="Each device has a master",
1132 onfail="Some devices don't have a master assigned" )
Jon Hall94fd0472014-12-08 11:52:42 -08001133
Jon Hall6aec96b2015-01-19 14:49:31 -08001134 main.step( "Check if switch roles are consistent across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001135 ONOS1Mastership = main.ONOScli1.roles()
1136 ONOS2Mastership = main.ONOScli2.roles()
1137 ONOS3Mastership = main.ONOScli3.roles()
1138 ONOS4Mastership = main.ONOScli4.roles()
1139 ONOS5Mastership = main.ONOScli5.roles()
1140 ONOS6Mastership = main.ONOScli6.roles()
1141 ONOS7Mastership = main.ONOScli7.roles()
Jon Hall8f89dda2015-01-22 16:03:33 -08001142 if "Error" in ONOS1Mastership or not ONOS1Mastership\
1143 or "Error" in ONOS2Mastership or not ONOS2Mastership\
1144 or "Error" in ONOS3Mastership or not ONOS3Mastership\
1145 or "Error" in ONOS4Mastership or not ONOS4Mastership\
1146 or "Error" in ONOS5Mastership or not ONOS5Mastership\
1147 or "Error" in ONOS6Mastership or not ONOS6Mastership\
1148 or "Error" in ONOS7Mastership or not ONOS7Mastership:
Jon Hall6aec96b2015-01-19 14:49:31 -08001149 main.log.error( "Error in getting ONOS mastership" )
1150 main.log.warn( "ONOS1 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001151 repr( ONOS1Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001152 main.log.warn( "ONOS2 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001153 repr( ONOS2Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001154 main.log.warn( "ONOS3 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001155 repr( ONOS3Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001156 main.log.warn( "ONOS4 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001157 repr( ONOS4Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001158 main.log.warn( "ONOS5 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001159 repr( ONOS5Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001160 main.log.warn( "ONOS6 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001161 repr( ONOS6Mastership ) )
Jon Hall6aec96b2015-01-19 14:49:31 -08001162 main.log.warn( "ONOS7 mastership response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001163 repr( ONOS7Mastership ) )
1164 consistentMastership = main.FALSE
1165 elif ONOS1Mastership == ONOS2Mastership\
1166 and ONOS1Mastership == ONOS3Mastership\
1167 and ONOS1Mastership == ONOS4Mastership\
1168 and ONOS1Mastership == ONOS5Mastership\
1169 and ONOS1Mastership == ONOS6Mastership\
1170 and ONOS1Mastership == ONOS7Mastership:
1171 consistentMastership = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001172 main.log.report(
1173 "Switch roles are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -05001174 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001175 main.log.warn( "ONOS1 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001176 json.loads( ONOS1Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001177 separators=( ',', ': ' ) ) )
1178 main.log.warn( "ONOS2 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001179 json.loads( ONOS2Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001180 separators=( ',', ': ' ) ) )
1181 main.log.warn( "ONOS3 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001182 json.loads( ONOS3Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001183 separators=( ',', ': ' ) ) )
1184 main.log.warn( "ONOS4 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001185 json.loads( ONOS4Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001186 separators=( ',', ': ' ) ) )
1187 main.log.warn( "ONOS5 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001188 json.loads( ONOS5Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001189 separators=( ',', ': ' ) ) )
1190 main.log.warn( "ONOS6 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001191 json.loads( ONOS6Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001192 separators=( ',', ': ' ) ) )
1193 main.log.warn( "ONOS7 roles: ", json.dumps(
Jon Hall8f89dda2015-01-22 16:03:33 -08001194 json.loads( ONOS7Mastership ), sort_keys=True, indent=4,
Jon Hall6aec96b2015-01-19 14:49:31 -08001195 separators=( ',', ': ' ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001196 consistentMastership = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001197 utilities.assert_equals(
1198 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001199 actual=consistentMastership,
Jon Hall6aec96b2015-01-19 14:49:31 -08001200 onpass="Switch roles are consistent across all ONOS nodes",
1201 onfail="ONOS nodes have different views of switch roles" )
Jon Hallb1290e82014-11-18 16:17:48 -05001202
1203 description2 = "Compare switch roles from before failure"
Jon Hall6aec96b2015-01-19 14:49:31 -08001204 main.step( description2 )
Jon Hallb1290e82014-11-18 16:17:48 -05001205
Jon Hall8f89dda2015-01-22 16:03:33 -08001206 currentJson = json.loads( ONOS1Mastership )
1207 oldJson = json.loads( mastershipState )
1208 mastershipCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001209 for i in range( 1, 29 ):
1210 switchDPID = str(
1211 main.Mininet1.getSwitchDPID(
1212 switch="s" +
1213 str( i ) ) )
Jon Hallb1290e82014-11-18 16:17:48 -05001214
Jon Hall8f89dda2015-01-22 16:03:33 -08001215 current = [ switch[ 'master' ] for switch in currentJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001216 if switchDPID in switch[ 'id' ] ]
Jon Hall8f89dda2015-01-22 16:03:33 -08001217 old = [ switch[ 'master' ] for switch in oldJson
Jon Hall6aec96b2015-01-19 14:49:31 -08001218 if switchDPID in switch[ 'id' ] ]
Jon Hallb1290e82014-11-18 16:17:48 -05001219 if current == old:
Jon Hall8f89dda2015-01-22 16:03:33 -08001220 mastershipCheck = mastershipCheck and main.TRUE
Jon Hallb1290e82014-11-18 16:17:48 -05001221 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001222 main.log.warn( "Mastership of switch %s changed" % switchDPID )
Jon Hall8f89dda2015-01-22 16:03:33 -08001223 mastershipCheck = main.FALSE
1224 if mastershipCheck == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001225 main.log.report( "Mastership of Switches was not changed" )
1226 utilities.assert_equals(
1227 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001228 actual=mastershipCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001229 onpass="Mastership of Switches was not changed",
1230 onfail="Mastership of some switches changed" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001231 mastershipCheck = mastershipCheck and consistentMastership
Jon Hallb1290e82014-11-18 16:17:48 -05001232
Jon Hall6aec96b2015-01-19 14:49:31 -08001233 main.step( "Get the intents and compare across all nodes" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001234 ONOS1Intents = main.ONOScli1.intents( jsonFormat=True )
1235 ONOS2Intents = main.ONOScli2.intents( jsonFormat=True )
1236 ONOS3Intents = main.ONOScli3.intents( jsonFormat=True )
1237 ONOS4Intents = main.ONOScli4.intents( jsonFormat=True )
1238 ONOS5Intents = main.ONOScli5.intents( jsonFormat=True )
1239 ONOS6Intents = main.ONOScli6.intents( jsonFormat=True )
1240 ONOS7Intents = main.ONOScli7.intents( jsonFormat=True )
1241 intentCheck = main.FALSE
1242 if "Error" in ONOS1Intents or not ONOS1Intents\
1243 or "Error" in ONOS2Intents or not ONOS2Intents\
1244 or "Error" in ONOS3Intents or not ONOS3Intents\
1245 or "Error" in ONOS4Intents or not ONOS4Intents\
1246 or "Error" in ONOS5Intents or not ONOS5Intents\
1247 or "Error" in ONOS6Intents or not ONOS6Intents\
1248 or "Error" in ONOS7Intents or not ONOS7Intents:
Jon Hall6aec96b2015-01-19 14:49:31 -08001249 main.log.report( "Error in getting ONOS intents" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001250 main.log.warn( "ONOS1 intents response: " + repr( ONOS1Intents ) )
1251 main.log.warn( "ONOS2 intents response: " + repr( ONOS2Intents ) )
1252 main.log.warn( "ONOS3 intents response: " + repr( ONOS3Intents ) )
1253 main.log.warn( "ONOS4 intents response: " + repr( ONOS4Intents ) )
1254 main.log.warn( "ONOS5 intents response: " + repr( ONOS5Intents ) )
1255 main.log.warn( "ONOS6 intents response: " + repr( ONOS6Intents ) )
1256 main.log.warn( "ONOS7 intents response: " + repr( ONOS7Intents ) )
1257 elif ONOS1Intents == ONOS2Intents\
1258 and ONOS1Intents == ONOS3Intents\
1259 and ONOS1Intents == ONOS4Intents\
1260 and ONOS1Intents == ONOS5Intents\
1261 and ONOS1Intents == ONOS6Intents\
1262 and ONOS1Intents == ONOS7Intents:
1263 intentCheck = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001264 main.log.report( "Intents are consistent across all ONOS nodes" )
Jon Hallb1290e82014-11-18 16:17:48 -05001265 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001266 main.log.warn( "ONOS1 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001267 print json.dumps( json.loads( ONOS1Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001268 indent=4, separators=( ',', ': ' ) )
1269 main.log.warn( "ONOS2 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001270 print json.dumps( json.loads( ONOS2Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001271 indent=4, separators=( ',', ': ' ) )
1272 main.log.warn( "ONOS3 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001273 print json.dumps( json.loads( ONOS3Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001274 indent=4, separators=( ',', ': ' ) )
1275 main.log.warn( "ONOS4 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001276 print json.dumps( json.loads( ONOS4Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001277 indent=4, separators=( ',', ': ' ) )
1278 main.log.warn( "ONOS5 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001279 print json.dumps( json.loads( ONOS5Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001280 indent=4, separators=( ',', ': ' ) )
1281 main.log.warn( "ONOS6 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001282 print json.dumps( json.loads( ONOS6Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001283 indent=4, separators=( ',', ': ' ) )
1284 main.log.warn( "ONOS7 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001285 print json.dumps( json.loads( ONOS7Intents ), sort_keys=True,
Jon Hall6aec96b2015-01-19 14:49:31 -08001286 indent=4, separators=( ',', ': ' ) )
1287 utilities.assert_equals(
1288 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001289 actual=intentCheck,
Jon Hall6aec96b2015-01-19 14:49:31 -08001290 onpass="Intents are consistent across all ONOS nodes",
1291 onfail="ONOS nodes have different views of intents" )
Jon Hall1b8f54a2015-02-04 13:24:20 -08001292 # Print the intent states
1293 intents = []
1294 intents.append( ONOS1Intents )
1295 intents.append( ONOS2Intents )
1296 intents.append( ONOS3Intents )
1297 intents.append( ONOS4Intents )
1298 intents.append( ONOS5Intents )
1299 intents.append( ONOS6Intents )
1300 intents.append( ONOS7Intents )
1301 intentStates = []
1302 for node in intents: # Iter through ONOS nodes
1303 nodeStates = []
1304 for intent in json.loads( node ): # Iter through intents of a node
1305 nodeStates.append( intent[ 'state' ] )
1306 intentStates.append( nodeStates )
1307 out = [ (i, nodeStates.count( i ) ) for i in set( nodeStates ) ]
1308 main.log.info( dict( out ) )
1309
Jon Hallb1290e82014-11-18 16:17:48 -05001310
Jon Hall6aec96b2015-01-19 14:49:31 -08001311 # NOTE: Hazelcast has no durability, so intents are lost across system
1312 # restarts
1313 main.step( "Compare current intents with intents before the failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001314 # NOTE: this requires case 5 to pass for intentState to be set.
Jon Hall94fd0472014-12-08 11:52:42 -08001315 # maybe we should stop the test if that fails?
Jon Hall1b8f54a2015-02-04 13:24:20 -08001316 sameIntents = main.TRUE
1317 if intentState and intentState == ONOS1Intents:
Jon Hall8f89dda2015-01-22 16:03:33 -08001318 sameIntents = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001319 main.log.report( "Intents are consistent with before failure" )
1320 # TODO: possibly the states have changed? we may need to figure out
1321 # what the aceptable states are
Jon Hallb1290e82014-11-18 16:17:48 -05001322 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001323 try:
1324 main.log.warn( "ONOS1 intents: " )
Jon Hall8f89dda2015-01-22 16:03:33 -08001325 print json.dumps( json.loads( ONOS1Intents ),
Jon Hall6aec96b2015-01-19 14:49:31 -08001326 sort_keys=True, indent=4,
1327 separators=( ',', ': ' ) )
1328 except:
1329 pass
Jon Hall8f89dda2015-01-22 16:03:33 -08001330 sameIntents = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001331 utilities.assert_equals(
1332 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001333 actual=sameIntents,
Jon Hall6aec96b2015-01-19 14:49:31 -08001334 onpass="Intents are consistent with before failure",
1335 onfail="The Intents changed during failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001336 intentCheck = intentCheck and sameIntents
Jon Hallb1290e82014-11-18 16:17:48 -05001337
Jon Hall6aec96b2015-01-19 14:49:31 -08001338 main.step( "Get the OF Table entries and compare to before " +
1339 "component failure" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001340 FlowTables = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001341 flows2 = []
1342 for i in range( 28 ):
1343 main.log.info( "Checking flow table on s" + str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001344 tmpFlows = main.Mininet2.getFlowTable( 1.3, "s" + str( i + 1 ) )
1345 flows2.append( tmpFlows )
1346 tempResult = main.Mininet2.flowComp(
Jon Hall6aec96b2015-01-19 14:49:31 -08001347 flow1=flows[ i ],
Jon Hall8f89dda2015-01-22 16:03:33 -08001348 flow2=tmpFlows )
1349 FlowTables = FlowTables and tempResult
1350 if FlowTables == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001351 main.log.info( "Differences in flow table for switch: s" +
1352 str( i + 1 ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001353 if FlowTables == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001354 main.log.report( "No changes were found in the flow tables" )
1355 utilities.assert_equals(
1356 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001357 actual=FlowTables,
Jon Hall6aec96b2015-01-19 14:49:31 -08001358 onpass="No changes were found in the flow tables",
1359 onfail="Changes were found in the flow tables" )
Jon Hallb1290e82014-11-18 16:17:48 -05001360
Jon Hall6aec96b2015-01-19 14:49:31 -08001361 main.step( "Check the continuous pings to ensure that no packets " +
1362 "were dropped during component failure" )
1363 # FIXME: This check is always failing. Investigate cause
1364 # NOTE: this may be something to do with file permsissions
Jon Hallb1290e82014-11-18 16:17:48 -05001365 # or slight change in format
Jon Hall6aec96b2015-01-19 14:49:31 -08001366 main.Mininet2.pingKill(
1367 main.params[ 'TESTONUSER' ],
1368 main.params[ 'TESTONIP' ] )
Jon Hall8f89dda2015-01-22 16:03:33 -08001369 LossInPings = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001370 # NOTE: checkForLoss returns main.FALSE with 0% packet loss
1371 for i in range( 8, 18 ):
1372 main.log.info(
1373 "Checking for a loss in pings along flow from s" +
1374 str( i ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001375 LossInPings = main.Mininet2.checkForLoss(
Jon Hall6aec96b2015-01-19 14:49:31 -08001376 "/tmp/ping.h" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001377 str( i ) ) or LossInPings
1378 if LossInPings == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001379 main.log.info( "Loss in ping detected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001380 elif LossInPings == main.ERROR:
Jon Hall6aec96b2015-01-19 14:49:31 -08001381 main.log.info( "There are multiple mininet process running" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001382 elif LossInPings == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001383 main.log.info( "No Loss in the pings" )
1384 main.log.report( "No loss of dataplane connectivity" )
1385 utilities.assert_equals(
1386 expect=main.FALSE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001387 actual=LossInPings,
Jon Hall6aec96b2015-01-19 14:49:31 -08001388 onpass="No Loss of connectivity",
1389 onfail="Loss of dataplane connectivity detected" )
Jon Hallb1290e82014-11-18 16:17:48 -05001390
Jon Hall6aec96b2015-01-19 14:49:31 -08001391 # Test of LeadershipElection
1392 # NOTE: this only works for the sanity test. In case of failures,
1393 # leader will likely change
Jon Hall8f89dda2015-01-22 16:03:33 -08001394 leader = ONOS1Ip
1395 leaderResult = main.TRUE
1396 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001397 # loop through ONOScli handlers
1398 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001399 leaderN = node.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001400 # verify leader is ONOS1
Jon Hall669173b2014-12-17 11:36:30 -08001401 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08001402 # all is well
1403 # NOTE: In failure scenario, this could be a new node, maybe
1404 # check != ONOS1
Jon Hall669173b2014-12-17 11:36:30 -08001405 pass
1406 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001407 # error in response
1408 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001409 "electionTestLeader function," +
Jon Hall6aec96b2015-01-19 14:49:31 -08001410 " check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001411 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001412 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08001413 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001414 main.log.report( "ONOS" + str( controller ) + " sees " +
1415 str( leaderN ) +
1416 " as the leader of the election app. " +
1417 "Leader should be " + str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001418 if leaderResult:
1419 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08001420 "view of leader across listeners and a new " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001421 "leader was re-elected if applicable )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001422 utilities.assert_equals(
1423 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001424 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001425 onpass="Leadership election passed",
1426 onfail="Something went wrong with Leadership election" )
Jon Hallb1290e82014-11-18 16:17:48 -05001427
Jon Hall8f89dda2015-01-22 16:03:33 -08001428 result = mastershipCheck and intentCheck and FlowTables and\
1429 ( not LossInPings ) and rolesNotNull and leaderResult
Jon Hall6aec96b2015-01-19 14:49:31 -08001430 result = int( result )
Jon Hallb1290e82014-11-18 16:17:48 -05001431 if result == main.TRUE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001432 main.log.report( "Constant State Tests Passed" )
1433 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall8f89dda2015-01-22 16:03:33 -08001434 onpass="Constant State Tests Passed",
1435 onfail="Constant state tests failed" )
Jon Hallb1290e82014-11-18 16:17:48 -05001436
Jon Hall6aec96b2015-01-19 14:49:31 -08001437 def CASE8( self, main ):
1438 """
Jon Hallb1290e82014-11-18 16:17:48 -05001439 Compare topo
Jon Hall6aec96b2015-01-19 14:49:31 -08001440 """
Jon Hallb1290e82014-11-18 16:17:48 -05001441 import sys
Jon Hall6aec96b2015-01-19 14:49:31 -08001442 # FIXME add this path to params
1443 sys.path.append( "/home/admin/sts" )
1444 # assumes that sts is already in you PYTHONPATH
1445 from sts.topology.teston_topology import TestONTopology
Jon Hallb1290e82014-11-18 16:17:48 -05001446 import json
Jon Hall73cf9cc2014-11-20 22:28:38 -08001447 import time
Jon Hallb1290e82014-11-18 16:17:48 -05001448
Jon Hall6aec96b2015-01-19 14:49:31 -08001449 description = "Compare ONOS Topology view to Mininet topology"
1450 main.case( description )
1451 main.log.report( description )
1452 main.step( "Create TestONTopology object" )
Jon Hallb1290e82014-11-18 16:17:48 -05001453 ctrls = []
1454 count = 1
1455 while True:
1456 temp = ()
Jon Hall6aec96b2015-01-19 14:49:31 -08001457 if ( 'ip' + str( count ) ) in main.params[ 'CTRL' ]:
1458 temp = temp + ( getattr( main, ( 'ONOS' + str( count ) ) ), )
1459 temp = temp + ( "ONOS" + str( count ), )
1460 temp = temp + ( main.params[ 'CTRL' ][ 'ip' + str( count ) ], )
1461 temp = temp + \
1462 ( eval( main.params[ 'CTRL' ][ 'port' + str( count ) ] ), )
1463 ctrls.append( temp )
Jon Hallb1290e82014-11-18 16:17:48 -05001464 count = count + 1
1465 else:
1466 break
Jon Hall6aec96b2015-01-19 14:49:31 -08001467 MNTopo = TestONTopology(
1468 main.Mininet1,
1469 ctrls ) # can also add Intent API info for intent operations
Jon Hallb1290e82014-11-18 16:17:48 -05001470
Jon Hall6aec96b2015-01-19 14:49:31 -08001471 main.step( "Comparing ONOS topology to MN" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001472 devicesResults = main.TRUE
1473 portsResults = main.TRUE
1474 linksResults = main.TRUE
1475 topoResult = main.FALSE
Jon Hall73cf9cc2014-11-20 22:28:38 -08001476 elapsed = 0
Jon Hallffb386d2014-11-21 13:43:38 -08001477 count = 0
Jon Hall6aec96b2015-01-19 14:49:31 -08001478 main.step( "Collecting topology information from ONOS" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001479 startTime = time.time()
Jon Hall21270ac2015-02-16 17:59:55 -08001480 # Give time for Gossip to work
Jon Hall8f89dda2015-01-22 16:03:33 -08001481 while topoResult == main.FALSE and elapsed < 60:
Jon Hallffb386d2014-11-21 13:43:38 -08001482 count = count + 1
Jon Hall94fd0472014-12-08 11:52:42 -08001483 if count > 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08001484 # TODO: Depricate STS usage
1485 MNTopo = TestONTopology(
1486 main.Mininet1,
1487 ctrls )
Jon Hall8f89dda2015-01-22 16:03:33 -08001488 cliStart = time.time()
Jon Hall94fd0472014-12-08 11:52:42 -08001489 devices = []
1490 devices.append( main.ONOScli1.devices() )
1491 devices.append( main.ONOScli2.devices() )
1492 devices.append( main.ONOScli3.devices() )
1493 devices.append( main.ONOScli4.devices() )
1494 devices.append( main.ONOScli5.devices() )
1495 devices.append( main.ONOScli6.devices() )
1496 devices.append( main.ONOScli7.devices() )
1497 hosts = []
Jon Hall529a37f2015-01-28 10:02:00 -08001498 hosts.append( json.loads( main.ONOScli1.hosts() ) )
1499 hosts.append( json.loads( main.ONOScli2.hosts() ) )
1500 hosts.append( json.loads( main.ONOScli3.hosts() ) )
1501 hosts.append( json.loads( main.ONOScli4.hosts() ) )
1502 hosts.append( json.loads( main.ONOScli5.hosts() ) )
1503 hosts.append( json.loads( main.ONOScli6.hosts() ) )
1504 hosts.append( json.loads( main.ONOScli7.hosts() ) )
1505 for controller in range( 0, len( hosts ) ):
1506 controllerStr = str( controller + 1 )
1507 for host in hosts[ controller ]:
Jon Hall529a37f2015-01-28 10:02:00 -08001508 if host[ 'ips' ] == []:
1509 main.log.error(
1510 "DEBUG:Error with host ips on controller" +
1511 controllerStr + ": " + str( host ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001512 ports = []
1513 ports.append( main.ONOScli1.ports() )
1514 ports.append( main.ONOScli2.ports() )
1515 ports.append( main.ONOScli3.ports() )
1516 ports.append( main.ONOScli4.ports() )
1517 ports.append( main.ONOScli5.ports() )
1518 ports.append( main.ONOScli6.ports() )
1519 ports.append( main.ONOScli7.ports() )
1520 links = []
1521 links.append( main.ONOScli1.links() )
1522 links.append( main.ONOScli2.links() )
1523 links.append( main.ONOScli3.links() )
1524 links.append( main.ONOScli4.links() )
1525 links.append( main.ONOScli5.links() )
1526 links.append( main.ONOScli6.links() )
1527 links.append( main.ONOScli7.links() )
1528 clusters = []
1529 clusters.append( main.ONOScli1.clusters() )
1530 clusters.append( main.ONOScli2.clusters() )
1531 clusters.append( main.ONOScli3.clusters() )
1532 clusters.append( main.ONOScli4.clusters() )
1533 clusters.append( main.ONOScli5.clusters() )
1534 clusters.append( main.ONOScli6.clusters() )
1535 clusters.append( main.ONOScli7.clusters() )
Jon Hallb1290e82014-11-18 16:17:48 -05001536
Jon Hall8f89dda2015-01-22 16:03:33 -08001537 elapsed = time.time() - startTime
1538 cliTime = time.time() - cliStart
1539 print "CLI time: " + str( cliTime )
Jon Hallb1290e82014-11-18 16:17:48 -05001540
Jon Hall21270ac2015-02-16 17:59:55 -08001541 for controller in range( numControllers ):
1542 controllerStr = str( controller + 1 )
1543 if devices[ controller ] or "Error" not in devices[
1544 controller ]:
1545 currentDevicesResult = main.Mininet1.compareSwitches(
1546 MNTopo,
1547 json.loads(
1548 devices[ controller ] ) )
1549 else:
1550 currentDevicesResult = main.FALSE
1551 utilities.assert_equals( expect=main.TRUE,
1552 actual=currentDevicesResult,
1553 onpass="ONOS" + controllerStr +
1554 " Switches view is correct",
1555 onfail="ONOS" + controllerStr +
1556 " Switches view is incorrect" )
Jon Hallb1290e82014-11-18 16:17:48 -05001557
Jon Hall21270ac2015-02-16 17:59:55 -08001558 if ports[ controller ] or "Error" not in ports[ controller ]:
1559 currentPortsResult = main.Mininet1.comparePorts(
1560 MNTopo,
1561 json.loads(
1562 ports[ controller ] ) )
1563 else:
1564 currentPortsResult = main.FALSE
1565 utilities.assert_equals( expect=main.TRUE,
1566 actual=currentPortsResult,
1567 onpass="ONOS" + controllerStr +
1568 " ports view is correct",
1569 onfail="ONOS" + controllerStr +
1570 " ports view is incorrect" )
Jon Hall94fd0472014-12-08 11:52:42 -08001571
Jon Hall21270ac2015-02-16 17:59:55 -08001572 if links[ controller ] or "Error" not in links[ controller ]:
1573 currentLinksResult = main.Mininet1.compareLinks(
1574 MNTopo,
1575 json.loads(
1576 links[ controller ] ) )
1577 else:
1578 currentLinksResult = main.FALSE
1579 utilities.assert_equals( expect=main.TRUE,
1580 actual=currentLinksResult,
1581 onpass="ONOS" + controllerStr +
1582 " links view is correct",
1583 onfail="ONOS" + controllerStr +
1584 " links view is incorrect" )
1585 devicesResults = devicesResults and currentDevicesResult
1586 portsResults = portsResults and currentPortsResult
1587 linksResults = linksResults and currentLinksResult
Jon Hall94fd0472014-12-08 11:52:42 -08001588
Jon Hall21270ac2015-02-16 17:59:55 -08001589 # Compare json objects for hosts and dataplane clusters
Jon Hall94fd0472014-12-08 11:52:42 -08001590
Jon Hall21270ac2015-02-16 17:59:55 -08001591 # hosts
1592 consistentHostsResult = main.TRUE
1593 for controller in range( len( hosts ) ):
1594 controllerStr = str( controller + 1 )
1595 if "Error" not in hosts[ controller ]:
1596 if hosts[ controller ] == hosts[ 0 ]:
1597 continue
1598 else: # hosts not consistent
1599 main.log.report( "hosts from ONOS" + controllerStr +
1600 " is inconsistent with ONOS1" )
1601 main.log.warn( repr( hosts[ controller ] ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001602 consistentHostsResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001603
Jon Hall21270ac2015-02-16 17:59:55 -08001604 else:
1605 main.log.report( "Error in getting ONOS hosts from ONOS" +
1606 controllerStr )
1607 consistentHostsResult = main.FALSE
1608 main.log.warn( "ONOS" + controllerStr +
1609 " hosts response: " +
1610 repr( hosts[ controller ] ) )
1611 utilities.assert_equals(
1612 expect=main.TRUE,
1613 actual=consistentHostsResult,
1614 onpass="Hosts view is consistent across all ONOS nodes",
1615 onfail="ONOS nodes have different views of hosts" )
Jon Hall94fd0472014-12-08 11:52:42 -08001616
Jon Hall21270ac2015-02-16 17:59:55 -08001617 # Strongly connected clusters of devices
1618 consistentClustersResult = main.TRUE
1619 for controller in range( len( clusters ) ):
1620 controllerStr = str( controller + 1 )
1621 if "Error" not in clusters[ controller ]:
1622 if clusters[ controller ] == clusters[ 0 ]:
1623 continue
1624 else: # clusters not consistent
1625 main.log.report( "clusters from ONOS" +
1626 controllerStr +
1627 " is inconsistent with ONOS1" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001628 consistentClustersResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001629
Jon Hall21270ac2015-02-16 17:59:55 -08001630 else:
1631 main.log.report( "Error in getting dataplane clusters " +
1632 "from ONOS" + controllerStr )
1633 consistentClustersResult = main.FALSE
1634 main.log.warn( "ONOS" + controllerStr +
1635 " clusters response: " +
1636 repr( clusters[ controller ] ) )
1637 utilities.assert_equals(
1638 expect=main.TRUE,
1639 actual=consistentClustersResult,
1640 onpass="Clusters view is consistent across all ONOS nodes",
1641 onfail="ONOS nodes have different views of clusters" )
1642 # there should always only be one cluster
1643 numClusters = len( json.loads( clusters[ 0 ] ) )
1644 utilities.assert_equals(
1645 expect=1,
1646 actual=numClusters,
1647 onpass="ONOS shows 1 SCC",
1648 onfail="ONOS shows " +
1649 str( numClusters ) +
1650 " SCCs" )
Jon Hall94fd0472014-12-08 11:52:42 -08001651
Jon Hall21270ac2015-02-16 17:59:55 -08001652 topoResult = ( devicesResults and portsResults and linksResults
1653 and consistentHostsResult
1654 and consistentClustersResult )
Jon Hall94fd0472014-12-08 11:52:42 -08001655
Jon Hall21270ac2015-02-16 17:59:55 -08001656 topoResult = topoResult and int( count <= 2 )
1657 note = "note it takes about " + str( int( cliTime ) ) + \
1658 " seconds for the test to make all the cli calls to fetch " +\
1659 "the topology from each ONOS instance"
1660 main.log.info(
1661 "Very crass estimate for topology discovery/convergence( " +
1662 str( note ) + " ): " + str( elapsed ) + " seconds, " +
1663 str( count ) + " tries" )
1664 utilities.assert_equals( expect=main.TRUE, actual=topoResult,
1665 onpass="Topology Check Test successful",
1666 onfail="Topology Check Test NOT successful" )
1667 if topoResult == main.TRUE:
1668 main.log.report( "ONOS topology view matches Mininet topology" )
Jon Hallb1290e82014-11-18 16:17:48 -05001669
Jon Hall6aec96b2015-01-19 14:49:31 -08001670 def CASE9( self, main ):
1671 """
Jon Hallb1290e82014-11-18 16:17:48 -05001672 Link s3-s28 down
Jon Hall6aec96b2015-01-19 14:49:31 -08001673 """
1674 import time
1675 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05001676
Jon Hall8f89dda2015-01-22 16:03:33 -08001677 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hallb1290e82014-11-18 16:17:48 -05001678
Jon Hall6aec96b2015-01-19 14:49:31 -08001679 description = "Turn off a link to ensure that Link Discovery " +\
Jon Hall8f89dda2015-01-22 16:03:33 -08001680 "is working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001681 main.log.report( description )
1682 main.case( description )
Jon Hallb1290e82014-11-18 16:17:48 -05001683
Jon Hall6aec96b2015-01-19 14:49:31 -08001684 main.step( "Kill Link between s3 and s28" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001685 LinkDown = main.Mininet1.link( END1="s3", END2="s28", OPTION="down" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001686 main.log.info(
1687 "Waiting " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001688 str( linkSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001689 " seconds for link down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001690 time.sleep( linkSleep )
1691 utilities.assert_equals( expect=main.TRUE, actual=LinkDown,
1692 onpass="Link down succesful",
1693 onfail="Failed to bring link down" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001694 # TODO do some sort of check here
Jon Hallb1290e82014-11-18 16:17:48 -05001695
Jon Hall6aec96b2015-01-19 14:49:31 -08001696 def CASE10( self, main ):
1697 """
Jon Hallb1290e82014-11-18 16:17:48 -05001698 Link s3-s28 up
Jon Hall6aec96b2015-01-19 14:49:31 -08001699 """
1700 import time
1701 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05001702
Jon Hall8f89dda2015-01-22 16:03:33 -08001703 linkSleep = float( main.params[ 'timers' ][ 'LinkDiscovery' ] )
Jon Hallb1290e82014-11-18 16:17:48 -05001704
Jon Hall6aec96b2015-01-19 14:49:31 -08001705 description = "Restore a link to ensure that Link Discovery is " + \
Jon Hall8f89dda2015-01-22 16:03:33 -08001706 "working properly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001707 main.log.report( description )
1708 main.case( description )
Jon Hallb1290e82014-11-18 16:17:48 -05001709
Jon Hall6aec96b2015-01-19 14:49:31 -08001710 main.step( "Bring link between s3 and s28 back up" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001711 LinkUp = main.Mininet1.link( END1="s3", END2="s28", OPTION="up" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001712 main.log.info(
1713 "Waiting " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001714 str( linkSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001715 " seconds for link up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001716 time.sleep( linkSleep )
1717 utilities.assert_equals( expect=main.TRUE, actual=LinkUp,
1718 onpass="Link up succesful",
1719 onfail="Failed to bring link up" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001720 # TODO do some sort of check here
Jon Hallb1290e82014-11-18 16:17:48 -05001721
Jon Hall6aec96b2015-01-19 14:49:31 -08001722 def CASE11( self, main ):
1723 """
Jon Hallb1290e82014-11-18 16:17:48 -05001724 Switch Down
Jon Hall6aec96b2015-01-19 14:49:31 -08001725 """
1726 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05001727 import time
1728
Jon Hall8f89dda2015-01-22 16:03:33 -08001729 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hallb1290e82014-11-18 16:17:48 -05001730
1731 description = "Killing a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001732 main.log.report( description )
1733 main.case( description )
1734 switch = main.params[ 'kill' ][ 'switch' ]
1735 switchDPID = main.params[ 'kill' ][ 'dpid' ]
Jon Hallb1290e82014-11-18 16:17:48 -05001736
Jon Hall6aec96b2015-01-19 14:49:31 -08001737 # TODO: Make this switch parameterizable
1738 main.step( "Kill " + switch )
1739 main.log.report( "Deleting " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08001740 main.Mininet1.delSwitch( switch )
1741 main.log.info( "Waiting " + str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001742 " seconds for switch down to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001743 time.sleep( switchSleep )
1744 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001745 # Peek at the deleted switch
1746 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001747 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001748 if device and device[ 'available' ] is False:
Jon Hall94fd0472014-12-08 11:52:42 -08001749 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001750 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall8f89dda2015-01-22 16:03:33 -08001751 onpass="Kill switch succesful",
1752 onfail="Failed to kill switch?" )
Jon Hallb1290e82014-11-18 16:17:48 -05001753
Jon Hall6aec96b2015-01-19 14:49:31 -08001754 def CASE12( self, main ):
1755 """
Jon Hallb1290e82014-11-18 16:17:48 -05001756 Switch Up
Jon Hall6aec96b2015-01-19 14:49:31 -08001757 """
1758 # NOTE: You should probably run a topology check after this
Jon Hallb1290e82014-11-18 16:17:48 -05001759 import time
Jon Hall669173b2014-12-17 11:36:30 -08001760
Jon Hall8f89dda2015-01-22 16:03:33 -08001761 switchSleep = float( main.params[ 'timers' ][ 'SwitchDiscovery' ] )
Jon Hall6aec96b2015-01-19 14:49:31 -08001762 switch = main.params[ 'kill' ][ 'switch' ]
1763 switchDPID = main.params[ 'kill' ][ 'dpid' ]
1764 links = main.params[ 'kill' ][ 'links' ].split()
Jon Hallb1290e82014-11-18 16:17:48 -05001765 description = "Adding a switch to ensure it is discovered correctly"
Jon Hall6aec96b2015-01-19 14:49:31 -08001766 main.log.report( description )
1767 main.case( description )
Jon Hallb1290e82014-11-18 16:17:48 -05001768
Jon Hall6aec96b2015-01-19 14:49:31 -08001769 main.step( "Add back " + switch )
1770 main.log.report( "Adding back " + switch )
Jon Hall8f89dda2015-01-22 16:03:33 -08001771 main.Mininet1.addSwitch( switch, dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001772 # TODO: New dpid or same? Ask Thomas?
1773 for peer in links:
Jon Hall8f89dda2015-01-22 16:03:33 -08001774 main.Mininet1.addLink( switch, peer )
1775 main.Mininet1.assignSwController(
Jon Hall6aec96b2015-01-19 14:49:31 -08001776 sw=switch.split( 's' )[ 1 ],
Jon Hall8f89dda2015-01-22 16:03:33 -08001777 count=numControllers,
1778 ip1=ONOS1Ip,
1779 port1=ONOS1Port,
1780 ip2=ONOS2Ip,
1781 port2=ONOS2Port,
1782 ip3=ONOS3Ip,
1783 port3=ONOS3Port,
1784 ip4=ONOS4Ip,
1785 port4=ONOS4Port,
1786 ip5=ONOS5Ip,
1787 port5=ONOS5Port,
1788 ip6=ONOS6Ip,
1789 port6=ONOS6Port,
1790 ip7=ONOS7Ip,
1791 port7=ONOS7Port )
Jon Hall6aec96b2015-01-19 14:49:31 -08001792 main.log.info(
1793 "Waiting " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001794 str( switchSleep ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001795 " seconds for switch up to be discovered" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001796 time.sleep( switchSleep )
1797 device = main.ONOScli1.getDevice( dpid=switchDPID )
Jon Hall6aec96b2015-01-19 14:49:31 -08001798 # Peek at the deleted switch
1799 main.log.warn( str( device ) )
Jon Hall94fd0472014-12-08 11:52:42 -08001800 result = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001801 if device and device[ 'available' ]:
Jon Hall94fd0472014-12-08 11:52:42 -08001802 result = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001803 utilities.assert_equals( expect=main.TRUE, actual=result,
Jon Hall8f89dda2015-01-22 16:03:33 -08001804 onpass="add switch succesful",
1805 onfail="Failed to add switch?" )
Jon Hallb1290e82014-11-18 16:17:48 -05001806
Jon Hall6aec96b2015-01-19 14:49:31 -08001807 def CASE13( self, main ):
1808 """
Jon Hallb1290e82014-11-18 16:17:48 -05001809 Clean up
Jon Hall6aec96b2015-01-19 14:49:31 -08001810 """
Jon Hall73cf9cc2014-11-20 22:28:38 -08001811 import os
1812 import time
Jon Hall6aec96b2015-01-19 14:49:31 -08001813 # TODO: make use of this elsewhere
1814 ips = []
Jon Hall8f89dda2015-01-22 16:03:33 -08001815 ips.append( ONOS1Ip )
1816 ips.append( ONOS2Ip )
1817 ips.append( ONOS3Ip )
1818 ips.append( ONOS4Ip )
1819 ips.append( ONOS5Ip )
1820 ips.append( ONOS6Ip )
1821 ips.append( ONOS7Ip )
Jon Hall6aec96b2015-01-19 14:49:31 -08001822
1823 # printing colors to terminal
Jon Hall669173b2014-12-17 11:36:30 -08001824 colors = {}
Jon Hall6aec96b2015-01-19 14:49:31 -08001825 colors[ 'cyan' ] = '\033[96m'
1826 colors[ 'purple' ] = '\033[95m'
1827 colors[ 'blue' ] = '\033[94m'
1828 colors[ 'green' ] = '\033[92m'
1829 colors[ 'yellow' ] = '\033[93m'
1830 colors[ 'red' ] = '\033[91m'
1831 colors[ 'end' ] = '\033[0m'
Jon Hall368769f2014-11-19 15:43:35 -08001832 description = "Test Cleanup"
Jon Hall6aec96b2015-01-19 14:49:31 -08001833 main.log.report( description )
1834 main.case( description )
1835 main.step( "Killing tcpdumps" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001836 main.Mininet2.stopTcpdump()
Jon Hallb1290e82014-11-18 16:17:48 -05001837
Jon Hall6aec96b2015-01-19 14:49:31 -08001838 main.step( "Checking ONOS Logs for errors" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001839 for i in range( 7 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001840 print colors[ 'purple' ] + "Checking logs for errors on " + \
Jon Hall8f89dda2015-01-22 16:03:33 -08001841 "ONOS" + str( i + 1 ) + ":" + colors[ 'end' ]
1842 print main.ONOSbench.checkLogs( ips[ i ] )
Jon Hall94fd0472014-12-08 11:52:42 -08001843
Jon Hall6aec96b2015-01-19 14:49:31 -08001844 main.step( "Copying MN pcap and ONOS log files to test station" )
Jon Hallb1290e82014-11-18 16:17:48 -05001845 testname = main.TEST
Jon Hall8f89dda2015-01-22 16:03:33 -08001846 teststationUser = main.params[ 'TESTONUSER' ]
1847 teststationIP = main.params[ 'TESTONIP' ]
Jon Hall6aec96b2015-01-19 14:49:31 -08001848 # NOTE: MN Pcap file is being saved to ~/packet_captures
Jon Hall73cf9cc2014-11-20 22:28:38 -08001849 # scp this file as MN and TestON aren't necessarily the same vm
Jon Hall6aec96b2015-01-19 14:49:31 -08001850 # FIXME: scp
1851 # mn files
1852 # TODO: Load these from params
1853 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08001854 logFolder = "/opt/onos/log/"
1855 logFiles = [ "karaf.log", "karaf.log.1" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08001856 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08001857 dstDir = "~/packet_captures/"
1858 for f in logFiles:
1859 for i in range( 7 ):
1860 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
1861 logFolder + f + " " +
1862 teststationUser + "@" +
1863 teststationIP + ":" +
1864 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001865 "-ONOS" + str( i + 1 ) + "-" +
1866 f )
1867 # std*.log's
1868 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08001869 logFolder = "/opt/onos/var/"
1870 logFiles = [ "stderr.log", "stdout.log" ]
Jon Hall6aec96b2015-01-19 14:49:31 -08001871 # NOTE: must end in /
Jon Hall8f89dda2015-01-22 16:03:33 -08001872 dstDir = "~/packet_captures/"
1873 for f in logFiles:
1874 for i in range( 7 ):
1875 main.ONOSbench.handle.sendline( "scp sdn@" + ips[ i ] + ":" +
1876 logFolder + f + " " +
1877 teststationUser + "@" +
1878 teststationIP + ":" +
1879 dstDir + str( testname ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001880 "-ONOS" + str( i + 1 ) + "-" +
1881 f )
1882 # sleep so scp can finish
1883 time.sleep( 10 )
1884 main.step( "Packing and rotating pcap archives" )
1885 os.system( "~/TestON/dependencies/rotate.sh " + str( testname ) )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001886
Jon Hall6aec96b2015-01-19 14:49:31 -08001887 # TODO: actually check something here
1888 utilities.assert_equals( expect=main.TRUE, actual=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001889 onpass="Test cleanup successful",
1890 onfail="Test cleanup NOT successful" )
Jon Hall73cf9cc2014-11-20 22:28:38 -08001891
Jon Hall6aec96b2015-01-19 14:49:31 -08001892 def CASE14( self, main ):
1893 """
Jon Hall94fd0472014-12-08 11:52:42 -08001894 start election app on all onos nodes
Jon Hall6aec96b2015-01-19 14:49:31 -08001895 """
Jon Hall8f89dda2015-01-22 16:03:33 -08001896 leaderResult = main.TRUE
Jon Hall6aec96b2015-01-19 14:49:31 -08001897 # install app on onos 1
1898 main.log.info( "Install leadership election app" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001899 main.ONOScli1.featureInstall( "onos-app-election" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001900 # wait for election
1901 # check for leader
Jon Hall8f89dda2015-01-22 16:03:33 -08001902 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001903 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08001904 if leader == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08001905 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08001906 pass
Jon Hall6aec96b2015-01-19 14:49:31 -08001907 elif leader is None:
1908 # No leader elected
1909 main.log.report( "No leader was elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001910 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001911 elif leader == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001912 # error in response
1913 # TODO: add check for "Command not found:" in the driver, this
1914 # means the app isn't loaded
Jon Hall8f89dda2015-01-22 16:03:33 -08001915 main.log.report( "Something is wrong with electionTestLeader" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001916 " function, check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001917 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001918 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08001919 # error in response
1920 main.log.report(
Jon Hall8f89dda2015-01-22 16:03:33 -08001921 "Unexpected response from electionTestLeader function:'" +
1922 str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001923 "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001924 leaderResult = main.FALSE
Jon Hall94fd0472014-12-08 11:52:42 -08001925
Jon Hall6aec96b2015-01-19 14:49:31 -08001926 # install on other nodes and check for leader.
1927 # Should be onos1 and each app should show the same leader
Jon Hall8f89dda2015-01-22 16:03:33 -08001928 for controller in range( 2, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08001929 # loop through ONOScli handlers
1930 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001931 node.featureInstall( "onos-app-election" )
1932 leaderN = node.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001933 # verify leader is ONOS1
Jon Hall8f89dda2015-01-22 16:03:33 -08001934 if leaderN == ONOS1Ip:
Jon Hall6aec96b2015-01-19 14:49:31 -08001935 # all is well
Jon Hall669173b2014-12-17 11:36:30 -08001936 pass
1937 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08001938 # error in response
1939 # TODO: add check for "Command not found:" in the driver, this
1940 # means the app isn't loaded
1941 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001942 "electionTestLeader function, check the" +
Jon Hall6aec96b2015-01-19 14:49:31 -08001943 " error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001944 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08001945 elif leader != leaderN:
Jon Hall8f89dda2015-01-22 16:03:33 -08001946 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08001947 main.log.report( "ONOS" + str( controller ) + " sees " +
1948 str( leaderN ) +
1949 " as the leader of the election app. Leader" +
1950 " should be " +
1951 str( leader ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08001952 if leaderResult:
Jon Hall6aec96b2015-01-19 14:49:31 -08001953 main.log.report( "Leadership election tests passed( consistent " +
1954 "view of leader across listeners and a leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08001955 "was elected )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001956 utilities.assert_equals(
1957 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001958 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001959 onpass="Leadership election passed",
1960 onfail="Something went wrong with Leadership election" )
Jon Hall94fd0472014-12-08 11:52:42 -08001961
Jon Hall6aec96b2015-01-19 14:49:31 -08001962 def CASE15( self, main ):
1963 """
Jon Hall669173b2014-12-17 11:36:30 -08001964 Check that Leadership Election is still functional
Jon Hall6aec96b2015-01-19 14:49:31 -08001965 """
Jon Hall8f89dda2015-01-22 16:03:33 -08001966 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08001967 description = "Check that Leadership Election is still functional"
Jon Hall6aec96b2015-01-19 14:49:31 -08001968 main.log.report( description )
1969 main.case( description )
1970 main.step( "Find current leader and withdraw" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001971 leader = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08001972 # TODO: do some sanity checking on leader before using it
Jon Hall8f89dda2015-01-22 16:03:33 -08001973 withdrawResult = main.FALSE
1974 if leader == ONOS1Ip:
1975 oldLeader = getattr( main, "ONOScli1" )
1976 elif leader == ONOS2Ip:
1977 oldLeader = getattr( main, "ONOScli2" )
1978 elif leader == ONOS3Ip:
1979 oldLeader = getattr( main, "ONOScli3" )
1980 elif leader == ONOS4Ip:
1981 oldLeader = getattr( main, "ONOScli4" )
1982 elif leader == ONOS5Ip:
1983 oldLeader = getattr( main, "ONOScli5" )
1984 elif leader == ONOS6Ip:
1985 oldLeader = getattr( main, "ONOScli6" )
1986 elif leader == ONOS7Ip:
1987 oldLeader = getattr( main, "ONOScli7" )
Jon Hall6aec96b2015-01-19 14:49:31 -08001988 elif leader is None or leader == main.FALSE:
1989 main.log.report(
1990 "Leader for the election app should be an ONOS node," +
1991 "instead got '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08001992 str( leader ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08001993 "'" )
Jon Hall8f89dda2015-01-22 16:03:33 -08001994 leaderResult = main.FALSE
1995 withdrawResult = oldLeader.electionTestWithdraw()
Jon Hall6aec96b2015-01-19 14:49:31 -08001996 utilities.assert_equals(
1997 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08001998 actual=withdrawResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08001999 onpass="App was withdrawn from election",
2000 onfail="App was not withdrawn from election" )
Jon Hall669173b2014-12-17 11:36:30 -08002001
Jon Hall6aec96b2015-01-19 14:49:31 -08002002 main.step( "Make sure new leader is elected" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002003 leaderList = []
2004 for controller in range( 1, numControllers + 1 ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002005 # loop through ONOScli handlers
2006 node = getattr( main, ( 'ONOScli' + str( controller ) ) )
Jon Hall8f89dda2015-01-22 16:03:33 -08002007 leaderList.append( node.electionTestLeader() )
2008 for leaderN in leaderList:
Jon Hall669173b2014-12-17 11:36:30 -08002009 if leaderN == leader:
Jon Hall6aec96b2015-01-19 14:49:31 -08002010 main.log.report(
2011 "ONOS" +
2012 str( controller ) +
2013 " still sees " +
2014 str( leader ) +
2015 " as leader after they withdrew" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002016 leaderResult = main.FALSE
Jon Hall669173b2014-12-17 11:36:30 -08002017 elif leaderN == main.FALSE:
Jon Hall6aec96b2015-01-19 14:49:31 -08002018 # error in response
2019 # TODO: add check for "Command not found:" in the driver, this
2020 # means the app isn't loaded
2021 main.log.report( "Something is wrong with " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002022 "electionTestLeader function, " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002023 "check the error logs" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002024 leaderResult = main.FALSE
2025 consistentLeader = main.FALSE
2026 if len( set( leaderList ) ) == 1:
Jon Hall6aec96b2015-01-19 14:49:31 -08002027 main.log.info( "Each Election-app sees '" +
Jon Hall8f89dda2015-01-22 16:03:33 -08002028 str( leaderList[ 0 ] ) +
Jon Hall6aec96b2015-01-19 14:49:31 -08002029 "' as the leader" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002030 consistentLeader = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002031 else:
Jon Hall6aec96b2015-01-19 14:49:31 -08002032 main.log.report(
2033 "Inconsistent responses for leader of Election-app:" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002034 for n in range( len( leaderList ) ):
Jon Hall6aec96b2015-01-19 14:49:31 -08002035 main.log.report( "ONOS" + str( n + 1 ) + " response: " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002036 str( leaderList[ n ] ) )
2037 if leaderResult:
2038 main.log.report( "Leadership election tests passed( consistent " +
Jon Hall6aec96b2015-01-19 14:49:31 -08002039 "view of leader across listeners and a new " +
2040 "leader was elected when the old leader " +
Jon Hall8f89dda2015-01-22 16:03:33 -08002041 "resigned )" )
Jon Hall6aec96b2015-01-19 14:49:31 -08002042 utilities.assert_equals(
2043 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002044 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002045 onpass="Leadership election passed",
2046 onfail="Something went wrong with Leadership election" )
Jon Hall669173b2014-12-17 11:36:30 -08002047
Jon Hall6aec96b2015-01-19 14:49:31 -08002048 main.step(
Jon Hall8f89dda2015-01-22 16:03:33 -08002049 "Run for election on old leader( just so everyone is in the hat )" )
2050 runResult = oldLeader.electionTestRun()
Jon Hall6aec96b2015-01-19 14:49:31 -08002051 utilities.assert_equals(
2052 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002053 actual=runResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002054 onpass="App re-ran for election",
2055 onfail="App failed to run for election" )
Jon Hall8f89dda2015-01-22 16:03:33 -08002056 if consistentLeader == main.TRUE:
2057 afterRun = main.ONOScli1.electionTestLeader()
Jon Hall6aec96b2015-01-19 14:49:31 -08002058 # verify leader didn't just change
Jon Hall8f89dda2015-01-22 16:03:33 -08002059 if afterRun == leaderList[ 0 ]:
2060 leaderResult = main.TRUE
Jon Hall669173b2014-12-17 11:36:30 -08002061 else:
Jon Hall8f89dda2015-01-22 16:03:33 -08002062 leaderResult = main.FALSE
Jon Hall6aec96b2015-01-19 14:49:31 -08002063 # TODO: assert on run and withdraw results?
Jon Hall669173b2014-12-17 11:36:30 -08002064
Jon Hall6aec96b2015-01-19 14:49:31 -08002065 utilities.assert_equals(
2066 expect=main.TRUE,
Jon Hall8f89dda2015-01-22 16:03:33 -08002067 actual=leaderResult,
Jon Hall6aec96b2015-01-19 14:49:31 -08002068 onpass="Leadership election passed",
2069 onfail="Something went wrong with Leadership election after " +
2070 "the old leader re-ran for election" )