admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 1 | |
| 2 | class RRCOnosSanity4nodesJ : |
| 3 | |
| 4 | def __init__(self) : |
| 5 | self.default = '' |
| 6 | |
| 7 | #***************************************************************************************************************************************************************************************** |
| 8 | #Test startup |
| 9 | #Tests the startup of Zookeeper1, RamCloud1, and ONOS1 to be certain that all started up successfully |
| 10 | def CASE1(self,main) : #Check to be sure ZK, Cass, and ONOS are up, then get ONOS version |
| 11 | import time |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 12 | main.case("Initial Setup") |
| 13 | main.step("stop onos") |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 14 | main.ONOS1.handle.sendline("cp ~/onos.properties.reactive ~/ONOS/conf/onos.properties") |
| 15 | main.ONOS2.handle.sendline("cp ~/onos.properties.reactive ~/ONOS/conf/onos.properties") |
| 16 | main.ONOS3.handle.sendline("cp ~/onos.properties.reactive ~/ONOS/conf/onos.properties") |
| 17 | main.ONOS4.handle.sendline("cp ~/onos.properties.reactive ~/ONOS/conf/onos.properties") |
admin | 12bae62 | 2014-06-05 09:44:30 -0700 | [diff] [blame] | 18 | main.ONOS1.stop_all() |
| 19 | main.ONOS2.stop_all() |
| 20 | main.ONOS3.stop_all() |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 21 | main.ONOS4.stop_all() |
| 22 | main.step("Start tcpdump on mn") |
| 23 | main.Mininet1.start_tcpdump(main.params['tcpdump']['filename'], intf = main.params['tcpdump']['intf'], port = main.params['tcpdump']['port']) |
| 24 | main.step("start ONOS") |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 25 | main.Zookeeper1.start() |
| 26 | main.Zookeeper2.start() |
| 27 | main.Zookeeper3.start() |
| 28 | main.Zookeeper4.start() |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 29 | main.RamCloud1.del_db() |
| 30 | main.RamCloud2.del_db() |
| 31 | main.RamCloud3.del_db() |
| 32 | main.RamCloud4.del_db() |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 33 | main.log.report("Pulling latest code from github to all nodes") |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 34 | for i in range(2): |
| 35 | uptodate = main.ONOS1.git_pull() |
| 36 | main.ONOS2.git_pull() |
| 37 | main.ONOS3.git_pull() |
| 38 | main.ONOS4.git_pull() |
| 39 | ver1 = main.ONOS1.get_version() |
| 40 | ver2 = main.ONOS4.get_version() |
| 41 | if ver1==ver2: |
| 42 | break |
| 43 | elif i==1: |
| 44 | main.ONOS2.git_pull("ONOS1 master") |
| 45 | main.ONOS3.git_pull("ONOS1 master") |
| 46 | main.ONOS4.git_pull("ONOS1 master") |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 47 | #if uptodate==0 |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 48 | if 1: |
| 49 | main.ONOS1.git_compile() |
| 50 | main.ONOS2.git_compile() |
| 51 | main.ONOS3.git_compile() |
| 52 | main.ONOS4.git_compile() |
| 53 | main.ONOS1.print_version() |
| 54 | # main.RamCloud1.git_pull() |
| 55 | # main.RamCloud2.git_pull() |
| 56 | # main.RamCloud3.git_pull() |
| 57 | # main.RamCloud4.git_pull() |
| 58 | # main.ONOS1.get_version() |
| 59 | # main.ONOS2.get_version() |
| 60 | # main.ONOS3.get_version() |
| 61 | # main.ONOS4.get_version() |
admin | 12bae62 | 2014-06-05 09:44:30 -0700 | [diff] [blame] | 62 | main.ONOS1.start_all() |
| 63 | main.ONOS2.start_all() |
| 64 | main.ONOS3.start_all() |
| 65 | main.ONOS4.start_all() |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 66 | main.ONOS2.start_rest() |
| 67 | time.sleep(10) |
| 68 | test= main.ONOS2.rest_status() |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 69 | if test == main.FALSE: |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 70 | main.ONOS2.start_rest() |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 71 | main.ONOS1.get_version() |
| 72 | main.log.report("Startup check Zookeeper1, RamCloud1, and ONOS1 connections") |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 73 | main.step("Testing startup Zookeeper") |
| 74 | data = main.Zookeeper1.isup() |
| 75 | utilities.assert_equals(expect=main.TRUE,actual=data,onpass="Zookeeper is up!",onfail="Zookeeper is down...") |
| 76 | main.step("Testing startup RamCloud") |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 77 | data = main.RamCloud1.status_serv() and main.RamCloud2.status_serv() and main.RamCloud3.status_serv() and main.RamCloud4.status_serv() |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 78 | if data == main.FALSE: |
| 79 | main.RamCloud1.stop_coor() |
| 80 | main.RamCloud1.stop_serv() |
| 81 | main.RamCloud2.stop_serv() |
| 82 | main.RamCloud3.stop_serv() |
| 83 | main.RamCloud4.stop_serv() |
| 84 | |
| 85 | time.sleep(5) |
| 86 | main.RamCloud1.start_coor() |
| 87 | main.RamCloud1.start_serv() |
| 88 | main.RamCloud2.start_serv() |
| 89 | main.RamCloud3.start_serv() |
| 90 | main.RamCloud4.start_serv() |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 91 | time.sleep(5) |
| 92 | data = main.RamCloud1.status_serv() and main.RamCloud2.status_serv() and main.RamCloud3.status_serv() and main.RamCloud4.status_serv() |
| 93 | |
| 94 | |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 95 | utilities.assert_equals(expect=main.TRUE,actual=data,onpass="RamCloud is up!",onfail="RamCloud is down...") |
| 96 | main.step("Testing startup ONOS") |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 97 | data = main.ONOS1.isup() and main.ONOS2.isup() and main.ONOS3.isup() and main.ONOS4.isup() |
| 98 | for i in range(3): |
| 99 | if data == main.FALSE: |
| 100 | #main.log.report("Something is funny... restarting ONOS") |
| 101 | #main.ONOS1.stop() |
| 102 | time.sleep(3) |
| 103 | #main.ONOS1.start() |
| 104 | #time.sleep(5) |
| 105 | data = main.ONOS1.isup() and main.ONOS2.isup() and main.ONOS3.isup() and main.ONOS4.isup() |
| 106 | else: |
| 107 | break |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 108 | utilities.assert_equals(expect=main.TRUE,actual=data,onpass="ONOS is up and running!",onfail="ONOS didn't start...") |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 109 | time.sleep(20) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 110 | |
| 111 | #********************************************************************************************************************************************************************************************** |
| 112 | #Assign Controllers |
| 113 | #This test first checks the ip of a mininet host, to be certain that the mininet exists(Host is defined in Params as <CASE1><destination>). |
| 114 | #Then the program assignes each ONOS instance a single controller to a switch(To be the initial master), then assigns all controllers. |
| 115 | #NOTE: The reason why all four controllers are assigned although one was already assigned as the master is due to the 'ovs-vsctl set-controller' command erases all present controllers if |
| 116 | # the controllers already assigned to the switch are not specified. |
| 117 | |
| 118 | def CASE2(self,main) : #Make sure mininet exists, then assign controllers to switches |
| 119 | import time |
| 120 | main.log.report("Check if mininet started properly, then assign controllers ONOS 1,2,3 and 4") |
| 121 | main.case("Checking if one MN host exists") |
| 122 | main.step("Host IP Checking using checkIP") |
| 123 | result = main.Mininet1.checkIP(main.params['CASE1']['destination']) |
| 124 | main.step("Verifying the result") |
| 125 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Host IP address configured",onfail="Host IP address not configured") |
| 126 | main.step("assigning ONOS controllers to switches") |
| 127 | for i in range(25): |
| 128 | if i < 3: |
| 129 | j=i+1 |
| 130 | main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1']) |
| 131 | time.sleep(1) |
| 132 | main.Mininet1.assign_sw_controller(sw=str(j),count=4,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4']) |
| 133 | elif i >= 3 and i < 5: |
| 134 | j=i+1 |
| 135 | main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip2'],port1=main.params['CTRL']['port2']) |
| 136 | time.sleep(1) |
| 137 | main.Mininet1.assign_sw_controller(sw=str(j),count=4,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4']) |
| 138 | elif i >= 5 and i < 15: |
| 139 | j=i+1 |
| 140 | main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip3'],port1=main.params['CTRL']['port3']) |
| 141 | time.sleep(1) |
| 142 | main.Mininet1.assign_sw_controller(sw=str(j),count=4,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4']) |
| 143 | else: |
| 144 | j=i+16 |
| 145 | main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip4'],port1=main.params['CTRL']['port4']) |
| 146 | time.sleep(1) |
| 147 | main.Mininet1.assign_sw_controller(sw=str(j),count=4,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4']) |
| 148 | main.Mininet1.get_sw_controller("s1") |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 149 | |
| 150 | # ********************************************************************************************************************************************************************************************** |
| 151 | #Add Flows |
| 152 | #Deletes any remnant flows from any previous test, add flows from the file labeled <FLOWDEF>, then runs the check flow test |
| 153 | #NOTE: THE FLOWDEF FILE MUST BE PRESENT ON TESTON VM!!! TestON will copy the file from its home machine into /tmp/flowtmp on the machine the ONOS instance is present on |
| 154 | |
| 155 | def CASE3(self,main) : #Delete any remnant flows, then add flows, and time how long it takes flow tables to update |
| 156 | main.log.report("Delete any flows from previous tests, then add flows from FLOWDEF file, then wait for switch flow tables to update") |
| 157 | import time |
| 158 | main.case("Taking care of these flows!") |
| 159 | main.step("Cleaning out any leftover flows...") |
| 160 | #main.ONOS1.delete_flow("all") |
| 161 | strtTime = time.time() |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 162 | main.ONOS1.rm_intents() |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 163 | print("world") |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 164 | main.ONOS1.add_intents() |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 165 | time.sleep(2) |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 166 | main.ONOS1.add_intents() |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 167 | print("hello") |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 168 | # main.ONOS1.add_flow(main.params['FLOWDEF']['testONip'],main.params['FLOWDEF']['user'],main.params['FLOWDEF']['password'],main.params['FLOWDEF']['flowDef']) |
| 169 | main.case("Checking flows") |
| 170 | |
| 171 | count = 1 |
| 172 | i = 6 |
| 173 | while i < 16 : |
| 174 | main.log.info("\n\t\t\t\th"+str(i)+" IS PINGING h"+str(i+25) ) |
| 175 | ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25)) |
| 176 | if ping == main.FALSE and count < 9: |
| 177 | count = count + 1 |
| 178 | i = 6 |
| 179 | main.log.info("Ping failed, making attempt number "+str(count)+" in 2 seconds") |
| 180 | time.sleep(2) |
| 181 | elif ping == main.FALSE and count ==9: |
| 182 | main.log.error("Ping test failed") |
| 183 | i = 17 |
| 184 | result = main.FALSE |
| 185 | elif ping == main.TRUE: |
| 186 | i = i + 1 |
| 187 | result = main.TRUE |
| 188 | endTime = time.time() |
| 189 | if result == main.TRUE: |
| 190 | main.log.report("\tTime to add flows: "+str(round(endTime-strtTime,2))+" seconds") |
| 191 | else: |
| 192 | main.log.report("\tFlows failed check") |
| 193 | |
| 194 | result2 = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 195 | main.step("Verifying the result") |
| 196 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Flow check PASS",onfail="Flow check FAIL") |
| 197 | |
| 198 | #********************************************************************************************************************************************************************************************** |
| 199 | #This test case removes Controllers 2,3, and 4 then performs a ping test. |
| 200 | #The assign controller is used because the ovs-vsctl module deletes all current controllers when a new controller is assigned. |
| 201 | #The ping test performs single pings on hosts from opposite sides of the topology. If one ping fails, the test waits 5 seconds before trying again. |
| 202 | #If the ping test fails 6 times, then the test case will return false |
| 203 | def CASE41(self,main) : |
| 204 | main.log.report("Testing Removal") |
| 205 | main.ONOS2.stop() |
| 206 | main.ONOS3.stop() |
| 207 | main.ONOS4.stop() |
| 208 | strtTime = time.time() |
| 209 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 210 | for i in range(10): |
| 211 | if result == main.FALSE: |
| 212 | time.sleep(5) |
| 213 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 214 | else: |
| 215 | break |
| 216 | |
| 217 | count = 1 |
| 218 | i = 6 |
| 219 | while i < 16 : |
| 220 | main.log.info("\n\t\t\t\th"+str(i)+" IS PINGING h"+str(i+25) ) |
| 221 | ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25)) |
| 222 | if ping == main.FALSE and count < 6: |
| 223 | count = count + 1 |
| 224 | i = 6 |
| 225 | main.log.info("Ping failed, making attempt number "+str(count)+" in 2 seconds") |
| 226 | time.sleep(2) |
| 227 | elif ping == main.FALSE and count ==6: |
| 228 | main.log.error("Ping test failed") |
| 229 | i = 17 |
| 230 | result = main.FALSE |
| 231 | elif ping == main.TRUE: |
| 232 | i = i + 1 |
| 233 | result = main.TRUE |
| 234 | endTime = time.time() |
| 235 | if result == main.TRUE: |
| 236 | main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds") |
| 237 | else: |
| 238 | main.log.report("\tPING TEST FAIL") |
| 239 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE") |
| 240 | time.sleep(10) |
| 241 | main.ONOS2.start() |
| 242 | main.ONOS3.start() |
| 243 | main.ONOS4.start() |
| 244 | time.sleep(10) |
| 245 | |
| 246 | |
| 247 | def CASE4(self,main) : |
| 248 | main.log.report("Remove ONOS 2,3,4 then ping until all hosts are reachable or fail after 6 attempts") |
| 249 | import time |
| 250 | for i in range(25): |
| 251 | if i < 15: |
| 252 | j=i+1 |
| 253 | main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1']) #Assigning a single controller removes all other controllers |
| 254 | else: |
| 255 | j=i+16 |
| 256 | main.Mininet1.assign_sw_controller(sw=str(j),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1']) |
| 257 | |
| 258 | strtTime = time.time() |
| 259 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 260 | for i in range(10): |
| 261 | if result == main.FALSE: |
| 262 | time.sleep(5) |
| 263 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 264 | else: |
| 265 | break |
| 266 | |
| 267 | count = 1 |
| 268 | i = 6 |
| 269 | while i < 16 : |
| 270 | main.log.info("\n\t\t\t\th"+str(i)+" IS PINGING h"+str(i+25) ) |
| 271 | ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25)) |
| 272 | if ping == main.FALSE and count < 6: |
| 273 | count = count + 1 |
| 274 | i = 6 |
| 275 | main.log.info("Ping failed, making attempt number "+str(count)+" in 2 seconds") |
| 276 | time.sleep(2) |
| 277 | elif ping == main.FALSE and count ==6: |
| 278 | main.log.error("Ping test failed") |
| 279 | i = 17 |
| 280 | result = main.FALSE |
| 281 | elif ping == main.TRUE: |
| 282 | i = i + 1 |
| 283 | result = main.TRUE |
| 284 | endTime = time.time() |
| 285 | if result == main.TRUE: |
| 286 | main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds") |
| 287 | else: |
| 288 | main.log.report("\tPING TEST FAIL") |
| 289 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE") |
| 290 | time.sleep(10) |
| 291 | |
| 292 | # ********************************************************************************************************************************************************************************************** |
| 293 | #This test case restores the controllers removed by Case 4 then performs a ping test. |
| 294 | |
| 295 | def CASE5(self,main) : |
| 296 | main.log.report("Restore ONOS 2,3,4 then ping until all hosts are reachable or fail after 6 attempts") |
| 297 | import time |
| 298 | for i in range(25): |
| 299 | if i < 15: |
| 300 | j=i+1 |
| 301 | main.Mininet1.assign_sw_controller(sw=str(j),count=4,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4']) |
| 302 | else: |
| 303 | j=i+16 |
| 304 | main.Mininet1.assign_sw_controller(sw=str(j),count=4,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4']) |
| 305 | |
| 306 | strtTime = time.time() |
| 307 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 308 | for i in range(10): |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 309 | if result == main.FALSE: |
| 310 | time.sleep(5) |
| 311 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 312 | else: |
| 313 | break |
| 314 | |
| 315 | count = 1 |
| 316 | i = 6 |
| 317 | while i < 16 : |
| 318 | main.log.info("\n\t\t\t\th"+str(i)+" IS PINGING h"+str(i+25) ) |
| 319 | ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25)) |
| 320 | if ping == main.FALSE and count < 6: |
| 321 | count = count + 1 |
| 322 | i = 6 |
| 323 | main.log.info("Ping failed, making attempt number "+str(count)+" in 2 seconds") |
| 324 | time.sleep(2) |
| 325 | elif ping == main.FALSE and count ==6: |
| 326 | main.log.error("Ping test failed") |
| 327 | i = 17 |
| 328 | result = main.FALSE |
| 329 | elif ping == main.TRUE: |
| 330 | i = i + 1 |
| 331 | result = main.TRUE |
| 332 | endTime = time.time() |
| 333 | if result == main.TRUE: |
| 334 | main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds") |
| 335 | else: |
| 336 | main.log.report("\tPING TEST FAILED") |
| 337 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE") |
| 338 | |
| 339 | # ********************************************************************************************************************************************************************************************** |
| 340 | #Brings a link that all flows pass through in the mininet down, then runs a ping test to view reroute time |
| 341 | |
| 342 | def CASE6(self,main) : |
| 343 | main.log.report("Bring Link between s1 and s2 down, then ping until all hosts are reachable or fail after 10 attempts") |
| 344 | import time |
| 345 | main.case("Bringing Link down... ") |
| 346 | result = main.Mininet1.link(END1=main.params['LINK']['begin'],END2=main.params['LINK']['end'],OPTION="down") |
| 347 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link DOWN!",onfail="Link not brought down...") |
| 348 | |
| 349 | strtTime = time.time() |
| 350 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],str(int(main.params['NR_Links'])-2)) |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 351 | for i in range(10): |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 352 | if result == main.FALSE: |
| 353 | time.sleep(5) |
| 354 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],str(int(main.params['NR_Links'])-2)) |
| 355 | else: |
| 356 | break |
| 357 | |
| 358 | count = 1 |
| 359 | i = 6 |
| 360 | while i < 16 : |
| 361 | main.log.info("\n\t\t\t\th"+str(i)+" IS PINGING h"+str(i+25) ) |
| 362 | ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25)) |
| 363 | if ping == main.FALSE and count < 10: |
| 364 | count = count + 1 |
| 365 | main.log.info("Ping failed, making attempt number "+str(count)+" in 2 seconds") |
| 366 | i = 6 |
| 367 | time.sleep(2) |
| 368 | elif ping == main.FALSE and count == 10: |
| 369 | main.log.error("Ping test failed") |
| 370 | i = 17 |
| 371 | result = main.FALSE |
| 372 | elif ping == main.TRUE: |
| 373 | i = i + 1 |
| 374 | result = main.TRUE |
| 375 | endTime = time.time() |
| 376 | if result == main.TRUE: |
| 377 | main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds") |
| 378 | else: |
| 379 | main.log.report("\tPING TEST FAILED") |
| 380 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE") |
| 381 | |
| 382 | # ********************************************************************************************************************************************************************************************** |
| 383 | #Brings the link that Case 6 took down back up, then runs a ping test to view reroute time |
| 384 | |
| 385 | def CASE7(self,main) : |
| 386 | main.log.report("Bring Link between s1 and s2 up, then ping until all hosts are reachable or fail after 10 attempts") |
| 387 | import time |
| 388 | main.case("Bringing Link up... ") |
| 389 | result = main.Mininet1.link(END1=main.params['LINK']['begin'],END2=main.params['LINK']['end'],OPTION="up") |
| 390 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Link UP!",onfail="Link not brought up...") |
| 391 | time.sleep(5) |
| 392 | strtTime = time.time() |
| 393 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 394 | for i in range(10): |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 395 | if result == main.FALSE: |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 396 | time.sleep(15) |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 397 | main.ONOS2.check_status_report(main.params['RestIP2'],main.params['NR_Switches'],main.params['NR_Links']) |
| 398 | main.ONOS3.check_status_report(main.params['RestIP3'],main.params['NR_Switches'],main.params['NR_Links']) |
| 399 | main.ONOS4.check_status_report(main.params['RestIP4'],main.params['NR_Switches'],main.params['NR_Links']) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 400 | result = main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 401 | else: |
| 402 | break |
| 403 | |
| 404 | strtTime = time.time() |
| 405 | count = 1 |
| 406 | i = 6 |
| 407 | while i < 16 : |
| 408 | main.log.info("\n\t\t\t\th"+str(i)+" IS PINGING h"+str(i+25) ) |
| 409 | ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25)) |
| 410 | if ping == main.FALSE and count < 10: |
| 411 | count = count + 1 |
| 412 | main.log.info("Ping failed, making attempt number "+str(count)+" in 2 seconds") |
| 413 | i = 6 |
| 414 | time.sleep(2) |
| 415 | elif ping == main.FALSE and count ==10: |
| 416 | main.log.error("Ping test failed") |
| 417 | i = 17 |
| 418 | result = main.FALSE |
| 419 | elif ping == main.TRUE: |
| 420 | i = i + 1 |
| 421 | result = main.TRUE |
| 422 | endTime = time.time() |
| 423 | if result == main.TRUE: |
| 424 | main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds") |
| 425 | else: |
| 426 | main.log.report("\tPING TESTS FAILED") |
| 427 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE") |
| 428 | |
| 429 | |
| 430 | # ****************************************************************************************************************************************************************** |
| 431 | # Test Device Discovery function by yanking s6:s6-eth0 interface and re-plug it into a switch |
| 432 | |
| 433 | def CASE21(self,main) : |
| 434 | import json |
| 435 | main.log.report("Test device discovery function, by attach, detach, move host h1 from s1->s6->s1. Per mininet naming, switch port the host attaches will remain as 's1-eth1' throughout the test.") |
| 436 | main.log.report("Check initially hostMAC/IP exist on the mininet...") |
| 437 | host = main.params['YANK']['hostname'] |
| 438 | mac = main.params['YANK']['hostmac'] |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 439 | RestIP1 = main.params['RESTCALL']['restIP1'] |
| 440 | RestPort = main.params['RESTCALL']['restPort'] |
| 441 | url = main.params['RESTCALL']['restURL'] |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 442 | |
| 443 | t_topowait = 5 |
admin | 12bae62 | 2014-06-05 09:44:30 -0700 | [diff] [blame] | 444 | t_restwait = 5 |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 445 | main.log.report( "Wait time from topo change to ping set to " + str(t_topowait)) |
| 446 | main.log.report( "Wait time from ping to rest call set to " + str(t_restwait)) |
| 447 | #print "host=" + host + "; RestIP=" + RestIP1 + "; RestPort=" + str(RestPort) |
| 448 | time.sleep(t_topowait) |
| 449 | main.log.info("\n\t\t\t\t ping issue one ping from " + str(host) + " to generate arp to switch. Ping result is not important" ) |
| 450 | ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254") |
| 451 | time.sleep(t_restwait) |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 452 | Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac) |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 453 | main.log.report("Number of host with MAC address = " + mac + " found by ONOS is: " + str(Reststatus)) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 454 | if Reststatus == 1: |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 455 | main.log.report("\t PASSED - Found host mac = " + mac + "; attached to switchDPID = " +"".join(Switch) + "; at port = " + str(Port[0])) |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 456 | result1 = main.TRUE |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 457 | elif Reststatus > 1: |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 458 | main.log.report("\t FAILED - Host " + host + " with MAC:" + mac + " has " + str(Reststatus) + " duplicated mac addresses. FAILED") |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 459 | main.log.report("switches are: " + "; ".join(Switch)) |
| 460 | main.log.report("Ports are: " + "; ".join(Port)) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 461 | result1 = main.FALSE |
| 462 | elif Reststatus == 0 and Switch == []: |
| 463 | main.log.report("\t FAILED - Host " + host + " with MAC:" + mac + " does not exist. FAILED") |
| 464 | result1 = main.FALSE |
| 465 | else:# check if rest server is working |
| 466 | main.log.error("Issue with find host") |
| 467 | result1 = main.FALSE |
| 468 | |
| 469 | |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 470 | ##### Step to yank out "s1-eth1" from s1, which is on autoONOS1 ##### |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 471 | |
| 472 | main.log.report("Yank out s1-eth1") |
| 473 | main.case("Yankout s6-eth1 (link to h1) from s1") |
| 474 | result = main.Mininet1.yank(SW=main.params['YANK']['sw1'],INTF=main.params['YANK']['intf']) |
| 475 | time.sleep(t_topowait) |
| 476 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Yank command suceeded",onfail="Yank command failed...") |
| 477 | |
| 478 | main.log.info("\n\t\t\t\t ping issue one ping from " + str(host) + " to generate arp to switch. Ping result is not important" ) |
| 479 | ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254") |
| 480 | time.sleep(t_restwait) |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 481 | Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 482 | |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 483 | main.log.report("Number of host with MAC = " + mac + " found by ONOS is: " + str(Reststatus)) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 484 | if Reststatus == 1: |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 485 | main.log.report("\tFAILED - Found host MAC = " + mac + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + str(Port)) |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 486 | result2 = main.FALSE |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 487 | elif Reststatus > 1: |
| 488 | main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " has " + str(Reststatus) + " duplicated IP addresses. FAILED") |
| 489 | main.log.report("switches are: " + "; ".join(Switch)) |
| 490 | main.log.report("Ports are: " + "; ".join(Port)) |
| 491 | main.log.report("MACs are: " + "; ".join(MAC)) |
| 492 | result2 = main.FALSE |
| 493 | elif Reststatus == 0 and Switch == []: |
| 494 | main.log.report("\t PASSED - Host " + host + " with MAC:" + str(mac) + " does not exist. PASSED - host is not supposed to be attached to the switch.") |
| 495 | result2 = main.TRUE |
| 496 | else:# check if rest server is working |
| 497 | main.log.error("Issue with find host") |
| 498 | result2 = main.FALSE |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 499 | |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 500 | ##### Step to plug "s1-eth1" to s6, which is on autoONOS3 ###### |
| 501 | main.log.report("Plug s1-eth1 into s6") |
| 502 | main.case("Plug s1-eth1 to s6") |
| 503 | result = main.Mininet1.plug(SW=main.params['PLUG']['sw6'],INTF=main.params['PLUG']['intf']) |
| 504 | time.sleep(t_topowait) |
| 505 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Plug command suceeded",onfail="Plug command failed...") |
| 506 | main.log.info("\n\t\t\t\t ping issue one ping from " + str(host) + " to generate arp to switch. Ping result is not important" ) |
| 507 | |
| 508 | ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254") |
| 509 | time.sleep(t_restwait) |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 510 | Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 511 | |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 512 | main.log.report("Number of host with MAC " + mac + " found by ONOS is: " + str(Reststatus)) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 513 | if Reststatus == 1: |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 514 | main.log.report("\tPASSED - Found host MAC = " + mac + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + str(Port[0])) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 515 | result3 = main.TRUE |
| 516 | elif Reststatus > 1: |
| 517 | main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " has " + str(Reststatus) + " duplicated IP addresses. FAILED") |
| 518 | main.log.report("switches are: " + "; ".join(Switch)) |
| 519 | main.log.report("Ports are: " + "; ".join(Port)) |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 520 | main.log.report("MACs are: " + "; ".join(MAC)) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 521 | result3 = main.FALSE |
| 522 | elif Reststatus == 0 and Switch == []: |
| 523 | main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " does not exist. FAILED") |
| 524 | result3 = main.FALSE |
| 525 | else:# check if rest server is working |
| 526 | main.log.error("Issue with find host") |
| 527 | result3 = main.FALSE |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 528 | |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 529 | ###### Step to put interface "s1-eth1" back to s1"##### |
| 530 | main.log.report("Move s1-eth1 back on to s1") |
| 531 | main.case("Move s1-eth1 back to s1") |
| 532 | result = main.Mininet1.yank(SW=main.params['YANK']['sw6'],INTF=main.params['YANK']['intf']) |
| 533 | time.sleep(t_topowait) |
| 534 | result = main.Mininet1.plug(SW=main.params['PLUG']['sw1'],INTF=main.params['PLUG']['intf']) |
| 535 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="Yank/Plug command suceeded",onfail="Yank/Plug command failed...") |
| 536 | main.log.info("\n\t\t\t\t ping issue one ping from " + str(host) + " to generate arp to switch. Ping result is not important" ) |
| 537 | |
| 538 | ping = main.Mininet1.pingHost(src = str(host),target = "10.0.0.254") |
| 539 | time.sleep(t_restwait) |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 540 | Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url, mac) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 541 | |
| 542 | main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus)) |
| 543 | if Reststatus == 1: |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 544 | main.log.report("\tPASSED - Found host MAC = " + mac + "; attached to switchDPID = " + "".join(Switch) + "; at port = " + str(Port[0])) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 545 | result4 = main.TRUE |
| 546 | elif Reststatus > 1: |
| 547 | main.log.report("\t FAILED - Host " + host + " with MAC:" + str(mac) + " has " + str(Reststatuas) + " duplicated IP addresses. FAILED") |
| 548 | main.log.report("switches are: " + "; ".join(Switch)) |
| 549 | main.log.report("Ports are: " + "; ".join(Port)) |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 550 | main.log.report("MACs are: " + "; ".join(MAC)) |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 551 | result4 = main.FALSE |
| 552 | elif Reststatus == 0 and Switch == []: |
| 553 | main.log.report("\t FAILED -Host " + host + " with MAC:" + str(mac) + " does not exist. FAILED") |
| 554 | result4 = main.FALSE |
| 555 | else:# check if rest server is working |
| 556 | main.log.error("Issue with find host") |
| 557 | result4 = main.FALSE |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 558 | time.sleep(20) |
| 559 | Reststatus, Switch, Port = main.ONOS1.find_host(RestIP1,RestPort,url,mac) |
| 560 | main.log.report("Number of host with IP=10.0.0.1 found by ONOS is: " + str(Reststatus)) |
| 561 | if Reststatus ==1: |
| 562 | main.log.report("\t FAILED - Host " + host + "with MAC:" + str(mac) + "was still found after expected timeout") |
| 563 | elif Reststatus>1: |
| 564 | main.log.report("\t FAILED - Host " + host + "with MAC:" + str(mac) + "was still found after expected timeout(multiple found)") |
| 565 | elif Reststatus==0: |
| 566 | main.log.report("\t PASSED - Device cleared after timeout") |
admin | f0713ab | 2014-04-29 10:43:35 -0700 | [diff] [blame] | 567 | |
| 568 | result = result1 and result2 and result3 and result4 |
| 569 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="DEVICE DISCOVERY TEST PASSED PLUG/UNPLUG/MOVE TEST",onfail="DEVICE DISCOVERY TEST FAILED") |
| 570 | |
| 571 | # Run a pure ping test. |
| 572 | |
| 573 | def CASE31(self, main): |
| 574 | main.log.report("Performing Ping Test") |
| 575 | count = 1 |
| 576 | i = 6 |
| 577 | while i < 16 : |
| 578 | main.log.info("\n\t\t\t\th"+str(i)+" IS PINGING h"+str(i+25) ) |
| 579 | strtTime = time.time() |
| 580 | ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+25)) |
| 581 | if ping == main.FALSE and count < 6: |
| 582 | count = count + 1 |
| 583 | i = 6 |
| 584 | main.log.info("Ping failed, making attempt number "+str(count)+" in 2 seconds") |
| 585 | time.sleep(2) |
| 586 | elif ping == main.FALSE and count ==6: |
| 587 | main.log.error("Ping test failed") |
| 588 | i = 17 |
| 589 | result = main.FALSE |
| 590 | elif ping == main.TRUE: |
| 591 | i = i + 1 |
| 592 | result = main.TRUE |
| 593 | endTime = time.time() |
| 594 | if result == main.TRUE: |
| 595 | main.log.report("\tTime to complete ping test: "+str(round(endTime-strtTime,2))+" seconds") |
| 596 | else: |
| 597 | main.log.report("\tPING TEST FAIL") |
| 598 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="NO PACKET LOSS, HOST IS REACHABLE",onfail="PACKET LOST, HOST IS NOT REACHABLE") |
| 599 | |
Jon Hall | 5a8aac6 | 2014-06-03 09:30:21 -0700 | [diff] [blame] | 600 | |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 601 | def CASE66(self, main): |
| 602 | main.log.report("Checking ONOS logs for exceptions") |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 603 | count = 0 |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 604 | check1 = main.ONOS1.check_exceptions() |
| 605 | main.log.report("Exceptions in ONOS1 logs: \n" + check1) |
| 606 | check2 = main.ONOS2.check_exceptions() |
| 607 | main.log.report("Exceptions in ONOS2 logs: \n" + check2) |
| 608 | check3 = main.ONOS3.check_exceptions() |
| 609 | main.log.report("Exceptions in ONOS3 logs: \n" + check3) |
| 610 | check4 = main.ONOS4.check_exceptions() |
| 611 | main.log.report("Exceptions in ONOS4 logs: \n" + check4) |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 612 | result = main.TRUE |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 613 | if (check1 or check2 or check3 or check4): |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 614 | result = main.FALSE |
| 615 | count = len(check1.splitlines()) + len(check2.splitlines()) + len(check3.splitlines()) + len(check4.splitlines()) |
| 616 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail=str(count) + " Exceptions were found in the logs") |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 617 | main.Mininet1.stop_tcpdump() |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 618 | |
| 619 | |