Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 1 | ''' |
| 2 | Description: This test is to determine if a single |
| 3 | instance ONOS 'cluster' can handle a restart |
| 4 | |
| 5 | List of test cases: |
| 6 | CASE1: Compile ONOS and push it to the test machines |
| 7 | CASE2: Assign mastership to controllers |
| 8 | CASE3: Assign intents |
| 9 | CASE4: Ping across added host intents |
| 10 | CASE5: Reading state of ONOS |
| 11 | CASE6: The Failure case. Since this is the Sanity test, we do nothing. |
| 12 | CASE7: Check state after control plane failure |
| 13 | CASE8: Compare topo |
| 14 | CASE9: Link s3-s28 down |
| 15 | CASE10: Link s3-s28 up |
| 16 | CASE11: Switch down |
| 17 | CASE12: Switch up |
| 18 | CASE13: Clean up |
| 19 | ''' |
| 20 | class SingleInstanceHATestRestart: |
| 21 | |
| 22 | def __init__(self) : |
| 23 | self.default = '' |
| 24 | |
| 25 | def CASE1(self,main) : |
| 26 | ''' |
| 27 | CASE1 is to compile ONOS and push it to the test machines |
| 28 | |
| 29 | Startup sequence: |
| 30 | git pull |
| 31 | mvn clean install |
| 32 | onos-package |
| 33 | cell <name> |
| 34 | onos-verify-cell |
| 35 | NOTE: temporary - onos-remove-raft-logs |
| 36 | onos-install -f |
| 37 | onos-wait-for-start |
| 38 | ''' |
| 39 | import time |
| 40 | main.log.report("ONOS Single node cluster restart HA test - initialization") |
| 41 | main.case("Setting up test environment") |
| 42 | |
| 43 | # load some vairables from the params file |
| 44 | PULL_CODE = False |
| 45 | if main.params['Git'] == 'True': |
| 46 | PULL_CODE = True |
| 47 | cell_name = main.params['ENV']['cellName'] |
| 48 | |
| 49 | #set global variables |
| 50 | global ONOS1_ip |
| 51 | global ONOS1_port |
| 52 | global ONOS2_ip |
| 53 | global ONOS2_port |
| 54 | global ONOS3_ip |
| 55 | global ONOS3_port |
| 56 | global ONOS4_ip |
| 57 | global ONOS4_port |
| 58 | global ONOS5_ip |
| 59 | global ONOS5_port |
| 60 | global ONOS6_ip |
| 61 | global ONOS6_port |
| 62 | global ONOS7_ip |
| 63 | global ONOS7_port |
| 64 | |
| 65 | ONOS1_ip = main.params['CTRL']['ip1'] |
| 66 | ONOS1_port = main.params['CTRL']['port1'] |
| 67 | ONOS2_ip = main.params['CTRL']['ip2'] |
| 68 | ONOS2_port = main.params['CTRL']['port2'] |
| 69 | ONOS3_ip = main.params['CTRL']['ip3'] |
| 70 | ONOS3_port = main.params['CTRL']['port3'] |
| 71 | ONOS4_ip = main.params['CTRL']['ip4'] |
| 72 | ONOS4_port = main.params['CTRL']['port4'] |
| 73 | ONOS5_ip = main.params['CTRL']['ip5'] |
| 74 | ONOS5_port = main.params['CTRL']['port5'] |
| 75 | ONOS6_ip = main.params['CTRL']['ip6'] |
| 76 | ONOS6_port = main.params['CTRL']['port6'] |
| 77 | ONOS7_ip = main.params['CTRL']['ip7'] |
| 78 | ONOS7_port = main.params['CTRL']['port7'] |
| 79 | |
| 80 | |
| 81 | main.step("Applying cell variable to environment") |
| 82 | cell_result = main.ONOSbench.set_cell(cell_name) |
| 83 | verify_result = main.ONOSbench.verify_cell() |
| 84 | |
| 85 | #FIXME:this is short term fix |
| 86 | main.log.report("Removing raft logs") |
| 87 | main.ONOSbench.onos_remove_raft_logs() |
| 88 | main.log.report("Uninstalling ONOS") |
| 89 | main.ONOSbench.onos_uninstall(ONOS1_ip) |
| 90 | main.ONOSbench.onos_uninstall(ONOS2_ip) |
| 91 | main.ONOSbench.onos_uninstall(ONOS3_ip) |
| 92 | main.ONOSbench.onos_uninstall(ONOS4_ip) |
| 93 | main.ONOSbench.onos_uninstall(ONOS5_ip) |
| 94 | main.ONOSbench.onos_uninstall(ONOS6_ip) |
| 95 | main.ONOSbench.onos_uninstall(ONOS7_ip) |
| 96 | |
| 97 | clean_install_result = main.TRUE |
| 98 | git_pull_result = main.TRUE |
| 99 | |
| 100 | main.step("Compiling the latest version of ONOS") |
| 101 | if PULL_CODE: |
| 102 | main.step("Git checkout and pull master") |
| 103 | main.ONOSbench.git_checkout("master") |
| 104 | git_pull_result = main.ONOSbench.git_pull() |
| 105 | |
| 106 | main.step("Using mvn clean & install") |
| 107 | clean_install_result = main.TRUE |
| 108 | if git_pull_result == main.TRUE: |
| 109 | clean_install_result = main.ONOSbench.clean_install() |
| 110 | else: |
| 111 | main.log.warn("Did not pull new code so skipping mvn "+ \ |
| 112 | "clean install") |
| 113 | main.ONOSbench.get_version(report=True) |
| 114 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 115 | cell_result = main.ONOSbench.set_cell("SingleHA") |
| 116 | verify_result = main.ONOSbench.verify_cell() |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 117 | main.step("Creating ONOS package") |
| 118 | package_result = main.ONOSbench.onos_package() |
| 119 | |
| 120 | main.step("Installing ONOS package") |
| 121 | onos1_install_result = main.ONOSbench.onos_install(options="-f", |
| 122 | node=ONOS1_ip) |
| 123 | |
| 124 | |
| 125 | main.step("Checking if ONOS is up yet") |
| 126 | #TODO: Refactor |
| 127 | # check bundle:list? |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 128 | for i in range(2): |
| 129 | onos1_isup = main.ONOSbench.isup(ONOS1_ip) |
| 130 | if onos1_isup: |
| 131 | break |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 132 | if not onos1_isup: |
| 133 | main.log.report("ONOS1 didn't start!") |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 134 | |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 135 | # TODO: if it becomes an issue, we can retry this step a few times |
| 136 | |
| 137 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 138 | cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 139 | |
| 140 | main.step("Start Packet Capture MN") |
| 141 | main.Mininet2.start_tcpdump( |
| 142 | str(main.params['MNtcpdump']['folder'])+str(main.TEST)+"-MN.pcap", |
| 143 | intf = main.params['MNtcpdump']['intf'], |
| 144 | port = main.params['MNtcpdump']['port']) |
| 145 | |
| 146 | |
| 147 | case1_result = (clean_install_result and package_result and |
| 148 | cell_result and verify_result and onos1_install_result and |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 149 | onos1_isup and cli_result) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 150 | |
| 151 | utilities.assert_equals(expect=main.TRUE, actual=case1_result, |
| 152 | onpass="Test startup successful", |
| 153 | onfail="Test startup NOT successful") |
| 154 | |
| 155 | |
| 156 | if case1_result==main.FALSE: |
| 157 | main.cleanup() |
| 158 | main.exit() |
| 159 | |
| 160 | def CASE2(self,main) : |
| 161 | ''' |
| 162 | Assign mastership to controllers |
| 163 | ''' |
| 164 | import time |
| 165 | import json |
| 166 | import re |
| 167 | |
| 168 | main.log.report("Assigning switches to controllers") |
| 169 | main.case("Assigning Controllers") |
| 170 | main.step("Assign switches to controllers") |
| 171 | |
| 172 | for i in range (1,29): |
| 173 | main.Mininet1.assign_sw_controller(sw=str(i), |
| 174 | ip1=ONOS1_ip,port1=ONOS1_port) |
| 175 | |
| 176 | mastership_check = main.TRUE |
| 177 | for i in range (1,29): |
| 178 | response = main.Mininet1.get_sw_controller("s"+str(i)) |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 179 | try: |
| 180 | main.log.info(str(response)) |
| 181 | except: |
| 182 | main.log.info(repr(response)) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 183 | if re.search("tcp:"+ONOS1_ip,response): |
| 184 | mastership_check = mastership_check and main.TRUE |
| 185 | else: |
| 186 | mastership_check = main.FALSE |
| 187 | if mastership_check == main.TRUE: |
| 188 | main.log.report("Switch mastership assigned correctly") |
| 189 | utilities.assert_equals(expect = main.TRUE,actual=mastership_check, |
| 190 | onpass="Switch mastership assigned correctly", |
| 191 | onfail="Switches not assigned correctly to controllers") |
| 192 | |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 193 | |
| 194 | |
| 195 | def CASE3(self,main) : |
| 196 | """ |
| 197 | Assign intents |
| 198 | |
| 199 | """ |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 200 | #FIXME: Only call this once when using a persistent datastore!!!! |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 201 | import time |
| 202 | import json |
| 203 | import re |
| 204 | main.log.report("Adding host intents") |
| 205 | main.case("Adding host Intents") |
| 206 | |
| 207 | main.step("Discovering Hosts( Via pingall for now)") |
| 208 | #FIXME: Once we have a host discovery mechanism, use that instead |
| 209 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 210 | #install onos-app-fwd |
| 211 | main.log.info("Install reactive forwarding app") |
| 212 | main.ONOScli1.feature_install("onos-app-fwd") |
| 213 | |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 214 | #REACTIVE FWD test |
| 215 | ping_result = main.FALSE |
| 216 | time1 = time.time() |
| 217 | ping_result = main.Mininet1.pingall() |
| 218 | time2 = time.time() |
| 219 | main.log.info("Time for pingall: %2f seconds" % (time2 - time1)) |
| 220 | |
| 221 | #uninstall onos-app-fwd |
| 222 | main.log.info("Uninstall reactive forwarding app") |
| 223 | main.ONOScli1.feature_uninstall("onos-app-fwd") |
| 224 | |
| 225 | main.step("Add host intents") |
| 226 | #TODO: move the host numbers to params |
| 227 | import json |
| 228 | intents_json= json.loads(main.ONOScli1.hosts()) |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 229 | intent_add_result = True |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 230 | for i in range(8,18): |
| 231 | main.log.info("Adding host intent between h"+str(i)+" and h"+str(i+10)) |
| 232 | host1 = "00:00:00:00:00:" + str(hex(i)[2:]).zfill(2).upper() |
| 233 | host2 = "00:00:00:00:00:" + str(hex(i+10)[2:]).zfill(2).upper() |
| 234 | #NOTE: get host can return None |
| 235 | #TODO: handle this |
| 236 | host1_id = main.ONOScli1.get_host(host1)['id'] |
| 237 | host2_id = main.ONOScli1.get_host(host2)['id'] |
| 238 | tmp_result = main.ONOScli1.add_host_intent(host1_id, host2_id ) |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 239 | intent_add_result = bool(intent_add_result and tmp_result) |
| 240 | utilities.assert_equals(expect=True, actual=intent_add_result, |
| 241 | onpass="Switch mastership correctly assigned", |
| 242 | onfail="Error in (re)assigning switch mastership") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 243 | #TODO Check if intents all exist in datastore |
| 244 | #NOTE: Do we need to print this once the test is working? |
| 245 | #main.log.info(json.dumps(json.loads(main.ONOScli1.intents(json_format=True)), |
| 246 | # sort_keys=True, indent=4, separators=(',', ': ') ) ) |
| 247 | |
| 248 | def CASE4(self,main) : |
| 249 | """ |
| 250 | Ping across added host intents |
| 251 | """ |
| 252 | description = " Ping across added host intents" |
| 253 | main.log.report(description) |
| 254 | main.case(description) |
| 255 | Ping_Result = main.TRUE |
| 256 | for i in range(8,18): |
| 257 | ping = main.Mininet1.pingHost(src="h"+str(i),target="h"+str(i+10)) |
| 258 | Ping_Result = Ping_Result and ping |
| 259 | if ping==main.FALSE: |
| 260 | main.log.warn("Ping failed between h"+str(i)+" and h" + str(i+10)) |
| 261 | elif ping==main.TRUE: |
| 262 | main.log.info("Ping test passed!") |
| 263 | Ping_Result = main.TRUE |
| 264 | if Ping_Result==main.FALSE: |
| 265 | main.log.report("Intents have not been installed correctly, pings failed.") |
| 266 | if Ping_Result==main.TRUE: |
| 267 | main.log.report("Intents have been installed correctly and verified by pings") |
| 268 | utilities.assert_equals(expect = main.TRUE,actual=Ping_Result, |
| 269 | onpass="Intents have been installed correctly and pings work", |
| 270 | onfail ="Intents have not been installed correctly, pings failed." ) |
| 271 | |
| 272 | def CASE5(self,main) : |
| 273 | ''' |
| 274 | Reading state of ONOS |
| 275 | ''' |
| 276 | import time |
| 277 | import json |
| 278 | from subprocess import Popen, PIPE |
| 279 | from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH |
| 280 | |
| 281 | main.log.report("Setting up and gathering data for current state") |
| 282 | main.case("Setting up and gathering data for current state") |
| 283 | #The general idea for this test case is to pull the state of (intents,flows, topology,...) from each ONOS node |
| 284 | #We can then compare them with eachother and also with past states |
| 285 | |
| 286 | main.step("Get the Mastership of each switch from each controller") |
| 287 | global mastership_state |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 288 | mastership_state = [] |
| 289 | |
| 290 | #Assert that each device has a master |
| 291 | roles_not_null = main.ONOScli1.roles_not_null() |
| 292 | utilities.assert_equals(expect = main.TRUE,actual=roles_not_null, |
| 293 | onpass="Each device has a master", |
| 294 | onfail="Some devices don't have a master assigned") |
| 295 | |
| 296 | |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 297 | ONOS1_mastership = main.ONOScli1.roles() |
| 298 | #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': ')) |
| 299 | #TODO: Make this a meaningful check |
| 300 | if "Error" in ONOS1_mastership or not ONOS1_mastership: |
| 301 | main.log.report("Error in getting ONOS roles") |
| 302 | main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership)) |
| 303 | consistent_mastership = main.FALSE |
| 304 | else: |
| 305 | mastership_state = ONOS1_mastership |
| 306 | consistent_mastership = main.TRUE |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 307 | |
| 308 | |
| 309 | main.step("Get the intents from each controller") |
| 310 | global intent_state |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 311 | intent_state = [] |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 312 | ONOS1_intents = main.ONOScli1.intents( json_format=True ) |
| 313 | intent_check = main.FALSE |
| 314 | if "Error" in ONOS1_intents or not ONOS1_intents: |
| 315 | main.log.report("Error in getting ONOS intents") |
| 316 | main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents)) |
| 317 | else: |
| 318 | intent_check = main.TRUE |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 319 | |
| 320 | |
| 321 | main.step("Get the flows from each controller") |
| 322 | global flow_state |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 323 | flow_state = [] |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 324 | ONOS1_flows = main.ONOScli1.flows( json_format=True ) |
| 325 | flow_check = main.FALSE |
| 326 | if "Error" in ONOS1_flows or not ONOS1_flows: |
| 327 | main.log.report("Error in getting ONOS intents") |
| 328 | main.log.warn("ONOS1 flows repsponse: "+ ONOS1_flows) |
| 329 | else: |
| 330 | #TODO: Do a better check, maybe compare flows on switches? |
| 331 | flow_state = ONOS1_flows |
| 332 | flow_check = main.TRUE |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 333 | |
| 334 | |
| 335 | main.step("Get the OF Table entries") |
| 336 | global flows |
| 337 | flows=[] |
| 338 | for i in range(1,29): |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 339 | flows.append(main.Mininet2.get_flowTable(1.3, "s"+str(i))) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 340 | |
| 341 | #TODO: Compare switch flow tables with ONOS flow tables |
| 342 | |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 343 | main.step("Create TestONTopology object") |
| 344 | ctrls = [] |
| 345 | count = 1 |
| 346 | temp = () |
| 347 | temp = temp + (getattr(main,('ONOS' + str(count))),) |
| 348 | temp = temp + ("ONOS"+str(count),) |
| 349 | temp = temp + (main.params['CTRL']['ip'+str(count)],) |
| 350 | temp = temp + (eval(main.params['CTRL']['port'+str(count)]),) |
| 351 | ctrls.append(temp) |
| 352 | MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations |
| 353 | |
| 354 | main.step("Collecting topology information from ONOS") |
| 355 | devices = [] |
| 356 | devices.append( main.ONOScli1.devices() ) |
| 357 | ''' |
| 358 | hosts = [] |
| 359 | hosts.append( main.ONOScli1.hosts() ) |
| 360 | ''' |
| 361 | ports = [] |
| 362 | ports.append( main.ONOScli1.ports() ) |
| 363 | links = [] |
| 364 | links.append( main.ONOScli1.links() ) |
| 365 | |
| 366 | |
| 367 | main.step("Comparing ONOS topology to MN") |
| 368 | devices_results = main.TRUE |
| 369 | ports_results = main.TRUE |
| 370 | links_results = main.TRUE |
| 371 | for controller in range(1): #TODO parameterize the number of controllers |
| 372 | if devices[controller] or not "Error" in devices[controller]: |
| 373 | current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller])) |
| 374 | else: |
| 375 | current_devices_result = main.FALSE |
| 376 | utilities.assert_equals(expect=main.TRUE, actual=current_devices_result, |
| 377 | onpass="ONOS"+str(int(controller+1))+" Switches view is correct", |
| 378 | onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect") |
| 379 | |
| 380 | if ports[controller] or not "Error" in ports[controller]: |
| 381 | current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller])) |
| 382 | else: |
| 383 | current_ports_result = main.FALSE |
| 384 | utilities.assert_equals(expect=main.TRUE, actual=current_ports_result, |
| 385 | onpass="ONOS"+str(int(controller+1))+" ports view is correct", |
| 386 | onfail="ONOS"+str(int(controller+1))+" ports view is incorrect") |
| 387 | |
| 388 | if links[controller] or not "Error" in links[controller]: |
| 389 | current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller])) |
| 390 | else: |
| 391 | current_links_result = main.FALSE |
| 392 | utilities.assert_equals(expect=main.TRUE, actual=current_links_result, |
| 393 | onpass="ONOS"+str(int(controller+1))+" links view is correct", |
| 394 | onfail="ONOS"+str(int(controller+1))+" links view is incorrect") |
| 395 | |
| 396 | devices_results = devices_results and current_devices_result |
| 397 | ports_results = ports_results and current_ports_result |
| 398 | links_results = links_results and current_links_result |
| 399 | |
| 400 | topo_result = devices_results and ports_results and links_results |
| 401 | utilities.assert_equals(expect=main.TRUE, actual=topo_result, |
| 402 | onpass="Topology Check Test successful", |
| 403 | onfail="Topology Check Test NOT successful") |
| 404 | |
| 405 | final_assert = main.TRUE |
| 406 | final_assert = final_assert and topo_result and flow_check \ |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 407 | and intent_check and consistent_mastership and roles_not_null |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 408 | utilities.assert_equals(expect=main.TRUE, actual=final_assert, |
| 409 | onpass="State check successful", |
| 410 | onfail="State check NOT successful") |
| 411 | |
| 412 | |
| 413 | def CASE6(self,main) : |
| 414 | ''' |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 415 | The Failure case. |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 416 | ''' |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 417 | import time |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 418 | |
| 419 | main.log.report("Restart ONOS node") |
| 420 | main.log.case("Restart ONOS node") |
| 421 | main.ONOSbench.onos_kill(ONOS1_ip) |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 422 | start = time.time() |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 423 | |
| 424 | main.step("Checking if ONOS is up yet") |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 425 | count = 0 |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 426 | while count < 10: |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 427 | onos1_isup = main.ONOSbench.isup(ONOS1_ip) |
| 428 | if onos1_isup == main.TRUE: |
| 429 | elapsed = time.time() - start |
| 430 | break |
| 431 | else: |
| 432 | count = count + 1 |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 433 | |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 434 | cli_result = main.ONOScli1.start_onos_cli(ONOS1_ip) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 435 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 436 | case_results = main.TRUE and onos1_isup and cli_result |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 437 | utilities.assert_equals(expect=main.TRUE, actual=case_results, |
| 438 | onpass="ONOS restart successful", |
| 439 | onfail="ONOS restart NOT successful") |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 440 | main.log.info("ESTIMATE: ONOS took %s seconds to restart" % str(elapsed) ) |
| 441 | time.sleep(5) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 442 | |
| 443 | def CASE7(self,main) : |
| 444 | ''' |
| 445 | Check state after ONOS failure |
| 446 | ''' |
| 447 | import os |
| 448 | import json |
| 449 | main.case("Running ONOS Constant State Tests") |
| 450 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 451 | #Assert that each device has a master |
| 452 | roles_not_null = main.ONOScli1.roles_not_null() |
| 453 | utilities.assert_equals(expect = main.TRUE,actual=roles_not_null, |
| 454 | onpass="Each device has a master", |
| 455 | onfail="Some devices don't have a master assigned") |
| 456 | |
| 457 | |
| 458 | |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 459 | main.step("Check if switch roles are consistent across all nodes") |
| 460 | ONOS1_mastership = main.ONOScli1.roles() |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 461 | #FIXME: Refactor this whole case for single instance |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 462 | #print json.dumps(json.loads(ONOS1_mastership), sort_keys=True, indent=4, separators=(',', ': ')) |
| 463 | if "Error" in ONOS1_mastership or not ONOS1_mastership: |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 464 | main.log.report("Error in getting ONOS mastership") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 465 | main.log.warn("ONOS1 mastership response: " + repr(ONOS1_mastership)) |
| 466 | consistent_mastership = main.FALSE |
| 467 | else: |
| 468 | consistent_mastership = main.TRUE |
| 469 | main.log.report("Switch roles are consistent across all ONOS nodes") |
| 470 | utilities.assert_equals(expect = main.TRUE,actual=consistent_mastership, |
| 471 | onpass="Switch roles are consistent across all ONOS nodes", |
| 472 | onfail="ONOS nodes have different views of switch roles") |
| 473 | |
| 474 | |
| 475 | description2 = "Compare switch roles from before failure" |
| 476 | main.step(description2) |
| 477 | |
| 478 | current_json = json.loads(ONOS1_mastership) |
| 479 | old_json = json.loads(mastership_state) |
| 480 | mastership_check = main.TRUE |
| 481 | for i in range(1,29): |
| 482 | switchDPID = str(main.Mininet1.getSwitchDPID(switch="s"+str(i))) |
| 483 | |
| 484 | current = [switch['master'] for switch in current_json if switchDPID in switch['id']] |
| 485 | old = [switch['master'] for switch in old_json if switchDPID in switch['id']] |
| 486 | if current == old: |
| 487 | mastership_check = mastership_check and main.TRUE |
| 488 | else: |
| 489 | main.log.warn("Mastership of switch %s changed" % switchDPID) |
| 490 | mastership_check = main.FALSE |
| 491 | if mastership_check == main.TRUE: |
| 492 | main.log.report("Mastership of Switches was not changed") |
| 493 | utilities.assert_equals(expect=main.TRUE,actual=mastership_check, |
| 494 | onpass="Mastership of Switches was not changed", |
| 495 | onfail="Mastership of some switches changed") |
| 496 | mastership_check = mastership_check and consistent_mastership |
| 497 | |
| 498 | |
| 499 | |
| 500 | main.step("Get the intents and compare across all nodes") |
| 501 | ONOS1_intents = main.ONOScli1.intents( json_format=True ) |
| 502 | intent_check = main.FALSE |
| 503 | if "Error" in ONOS1_intents or not ONOS1_intents: |
| 504 | main.log.report("Error in getting ONOS intents") |
| 505 | main.log.warn("ONOS1 intents response: " + repr(ONOS1_intents)) |
| 506 | else: |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 507 | intent_check = main.TRUE |
| 508 | main.log.report("Intents are consistent across all ONOS nodes") |
| 509 | utilities.assert_equals(expect = main.TRUE,actual=intent_check, |
| 510 | onpass="Intents are consistent across all ONOS nodes", |
| 511 | onfail="ONOS nodes have different views of intents") |
| 512 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 513 | #NOTE: Hazelcast has no durability, so intents are lost |
| 514 | ''' |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 515 | main.step("Compare current intents with intents before the failure") |
| 516 | if intent_state == ONOS1_intents: |
| 517 | same_intents = main.TRUE |
| 518 | main.log.report("Intents are consistent with before failure") |
| 519 | #TODO: possibly the states have changed? we may need to figure out what the aceptable states are |
| 520 | else: |
| 521 | same_intents = main.FALSE |
| 522 | utilities.assert_equals(expect = main.TRUE,actual=same_intents, |
| 523 | onpass="Intents are consistent with before failure", |
| 524 | onfail="The Intents changed during failure") |
| 525 | intent_check = intent_check and same_intents |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 526 | ''' |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 527 | |
| 528 | |
| 529 | |
| 530 | main.step("Get the OF Table entries and compare to before component failure") |
| 531 | Flow_Tables = main.TRUE |
| 532 | flows2=[] |
| 533 | for i in range(28): |
| 534 | main.log.info("Checking flow table on s" + str(i+1)) |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 535 | tmp_flows = main.Mininet2.get_flowTable(1.3, "s"+str(i+1)) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 536 | flows2.append(tmp_flows) |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 537 | temp_result = main.Mininet2.flow_comp(flow1=flows[i],flow2=tmp_flows) |
| 538 | Flow_Tables = Flow_Tables and temp_result |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 539 | if Flow_Tables == main.FALSE: |
| 540 | main.log.info("Differences in flow table for switch: "+str(i+1)) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 541 | if Flow_Tables == main.TRUE: |
| 542 | main.log.report("No changes were found in the flow tables") |
| 543 | utilities.assert_equals(expect=main.TRUE,actual=Flow_Tables, |
| 544 | onpass="No changes were found in the flow tables", |
| 545 | onfail="Changes were found in the flow tables") |
| 546 | |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 547 | #TODO:add topology to this or leave as a seperate case? |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 548 | result = mastership_check and intent_check and Flow_Tables and roles_not_null |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 549 | result = int(result) |
| 550 | if result == main.TRUE: |
| 551 | main.log.report("Constant State Tests Passed") |
| 552 | utilities.assert_equals(expect=main.TRUE,actual=result, |
| 553 | onpass="Constant State Tests Passed", |
| 554 | onfail="Constant state tests failed") |
| 555 | |
| 556 | def CASE8 (self,main): |
| 557 | ''' |
| 558 | Compare topo |
| 559 | ''' |
| 560 | import sys |
| 561 | sys.path.append("/home/admin/sts") # Trying to remove some dependancies, #FIXME add this path to params |
| 562 | from sts.topology.teston_topology import TestONTopology # assumes that sts is already in you PYTHONPATH |
| 563 | import json |
| 564 | import time |
| 565 | |
| 566 | description ="Compare ONOS Topology view to Mininet topology" |
| 567 | main.case(description) |
| 568 | main.log.report(description) |
| 569 | main.step("Create TestONTopology object") |
| 570 | ctrls = [] |
| 571 | count = 1 |
| 572 | temp = () |
| 573 | temp = temp + (getattr(main,('ONOS' + str(count))),) |
| 574 | temp = temp + ("ONOS"+str(count),) |
| 575 | temp = temp + (main.params['CTRL']['ip'+str(count)],) |
| 576 | temp = temp + (eval(main.params['CTRL']['port'+str(count)]),) |
| 577 | ctrls.append(temp) |
| 578 | MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations |
| 579 | |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 580 | main.step("Comparing ONOS topology to MN") |
| 581 | devices_results = main.TRUE |
| 582 | ports_results = main.TRUE |
| 583 | links_results = main.TRUE |
| 584 | topo_result = main.FALSE |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 585 | elapsed = 0 |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 586 | count = 0 |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 587 | main.step("Collecting topology information from ONOS") |
| 588 | start_time = time.time() |
| 589 | while topo_result == main.FALSE and elapsed < 60: |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 590 | count = count + 1 |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 591 | if count > 1: |
| 592 | MNTopo = TestONTopology(main.Mininet1, ctrls) # can also add Intent API info for intent operations |
| 593 | cli_start = time.time() |
| 594 | devices = [] |
| 595 | devices.append( main.ONOScli1.devices() ) |
| 596 | ''' |
| 597 | hosts = [] |
| 598 | hosts.append( main.ONOScli1.hosts() ) |
| 599 | ''' |
| 600 | ports = [] |
| 601 | ports.append( main.ONOScli1.ports() ) |
| 602 | links = [] |
| 603 | links.append( main.ONOScli1.links() ) |
| 604 | elapsed = time.time() - start_time |
| 605 | print "CLI time: " + str(time.time() - cli_start) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 606 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 607 | for controller in range(1): #TODO parameterize the number of controllers |
| 608 | if devices[controller] or not "Error" in devices[controller]: |
| 609 | current_devices_result = main.Mininet1.compare_switches(MNTopo, json.loads(devices[controller])) |
| 610 | else: |
| 611 | current_devices_result = main.FALSE |
| 612 | utilities.assert_equals(expect=main.TRUE, actual=current_devices_result, |
| 613 | onpass="ONOS"+str(int(controller+1))+" Switches view is correct", |
| 614 | onfail="ONOS"+str(int(controller+1))+" Switches view is incorrect") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 615 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 616 | if ports[controller] or not "Error" in ports[controller]: |
| 617 | current_ports_result = main.Mininet1.compare_ports(MNTopo, json.loads(ports[controller])) |
| 618 | else: |
| 619 | current_ports_result = main.FALSE |
| 620 | utilities.assert_equals(expect=main.TRUE, actual=current_ports_result, |
| 621 | onpass="ONOS"+str(int(controller+1))+" ports view is correct", |
| 622 | onfail="ONOS"+str(int(controller+1))+" ports view is incorrect") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 623 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 624 | if links[controller] or not "Error" in links[controller]: |
| 625 | current_links_result = main.Mininet1.compare_links(MNTopo, json.loads(links[controller])) |
| 626 | else: |
| 627 | current_links_result = main.FALSE |
| 628 | utilities.assert_equals(expect=main.TRUE, actual=current_links_result, |
| 629 | onpass="ONOS"+str(int(controller+1))+" links view is correct", |
| 630 | onfail="ONOS"+str(int(controller+1))+" links view is incorrect") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 631 | devices_results = devices_results and current_devices_result |
| 632 | ports_results = ports_results and current_ports_result |
| 633 | links_results = links_results and current_links_result |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 634 | topo_result = devices_results and ports_results and links_results |
| 635 | |
| 636 | topo_result = topo_result and int(count <= 2) |
| 637 | main.log.report("Very crass estimate for topology discovery/convergence(note it takes about 1 seconds to read the topology from each ONOS instance): " +\ |
Jon Hall | ffb386d | 2014-11-21 13:43:38 -0800 | [diff] [blame] | 638 | str(elapsed) + " seconds, " + str(count) +" tries" ) |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 639 | if elapsed > 60: |
| 640 | main.log.report("Giving up on topology convergence") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 641 | utilities.assert_equals(expect=main.TRUE, actual=topo_result, |
| 642 | onpass="Topology Check Test successful", |
| 643 | onfail="Topology Check Test NOT successful") |
| 644 | if topo_result == main.TRUE: |
| 645 | main.log.report("ONOS topology view matches Mininet topology") |
| 646 | |
| 647 | |
| 648 | def CASE9 (self,main): |
| 649 | ''' |
| 650 | Link s3-s28 down |
| 651 | ''' |
| 652 | #NOTE: You should probably run a topology check after this |
| 653 | |
| 654 | link_sleep = int(main.params['timers']['LinkDiscovery']) |
| 655 | |
| 656 | description = "Turn off a link to ensure that Link Discovery is working properly" |
| 657 | main.log.report(description) |
| 658 | main.case(description) |
| 659 | |
| 660 | |
| 661 | main.step("Kill Link between s3 and s28") |
| 662 | Link_Down = main.Mininet1.link(END1="s3",END2="s28",OPTION="down") |
| 663 | main.log.info("Waiting " + str(link_sleep) + " seconds for link down to be discovered") |
| 664 | time.sleep(link_sleep) |
| 665 | utilities.assert_equals(expect=main.TRUE,actual=Link_Down, |
| 666 | onpass="Link down succesful", |
| 667 | onfail="Failed to bring link down") |
| 668 | #TODO do some sort of check here |
| 669 | |
| 670 | def CASE10 (self,main): |
| 671 | ''' |
| 672 | Link s3-s28 up |
| 673 | ''' |
| 674 | #NOTE: You should probably run a topology check after this |
| 675 | |
| 676 | link_sleep = int(main.params['timers']['LinkDiscovery']) |
| 677 | |
| 678 | description = "Restore a link to ensure that Link Discovery is working properly" |
| 679 | main.log.report(description) |
| 680 | main.case(description) |
| 681 | |
| 682 | main.step("Bring link between s3 and s28 back up") |
| 683 | Link_Up = main.Mininet1.link(END1="s3",END2="s28",OPTION="up") |
| 684 | main.log.info("Waiting " + str(link_sleep) + " seconds for link up to be discovered") |
| 685 | time.sleep(link_sleep) |
| 686 | utilities.assert_equals(expect=main.TRUE,actual=Link_Up, |
| 687 | onpass="Link up succesful", |
| 688 | onfail="Failed to bring link up") |
| 689 | #TODO do some sort of check here |
| 690 | |
| 691 | |
| 692 | def CASE11 (self, main) : |
| 693 | ''' |
| 694 | Switch Down |
| 695 | ''' |
| 696 | #NOTE: You should probably run a topology check after this |
| 697 | import time |
| 698 | |
| 699 | switch_sleep = int(main.params['timers']['SwitchDiscovery']) |
| 700 | |
| 701 | description = "Killing a switch to ensure it is discovered correctly" |
| 702 | main.log.report(description) |
| 703 | main.case(description) |
| 704 | |
| 705 | #TODO: Make this switch parameterizable |
| 706 | main.step("Kill s28 ") |
| 707 | main.log.report("Deleting s28") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 708 | main.Mininet1.del_switch("s28") |
| 709 | main.log.info("Waiting " + str(switch_sleep) + " seconds for switch down to be discovered") |
| 710 | time.sleep(switch_sleep) |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 711 | device = main.ONOScli1.get_device(dpid="0028") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 712 | #Peek at the deleted switch |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 713 | main.log.warn( str(device) ) |
| 714 | result = main.FALSE |
| 715 | if device and device['available'] == False: |
| 716 | result = main.TRUE |
| 717 | utilities.assert_equals(expect=main.TRUE,actual=result, |
| 718 | onpass="Kill switch succesful", |
| 719 | onfail="Failed to kill switch?") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 720 | |
| 721 | def CASE12 (self, main) : |
| 722 | ''' |
| 723 | Switch Up |
| 724 | ''' |
| 725 | #NOTE: You should probably run a topology check after this |
| 726 | import time |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 727 | description = "Adding a switch to ensure it is discovered correctly" |
| 728 | main.log.report(description) |
| 729 | main.case(description) |
| 730 | |
| 731 | main.step("Add back s28") |
| 732 | main.log.report("Adding back s28") |
| 733 | main.Mininet1.add_switch("s28", dpid = '0000000000002800') |
| 734 | #TODO: New dpid or same? Ask Thomas? |
| 735 | main.Mininet1.add_link('s28', 's3') |
| 736 | main.Mininet1.add_link('s28', 's6') |
| 737 | main.Mininet1.add_link('s28', 'h28') |
| 738 | main.Mininet1.assign_sw_controller(sw="28", |
| 739 | ip1=ONOS1_ip,port1=ONOS1_port) |
| 740 | main.log.info("Waiting " + str(switch_sleep) + " seconds for switch up to be discovered") |
| 741 | time.sleep(switch_sleep) |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 742 | device = main.ONOScli1.get_device(dpid="0028") |
| 743 | #Peek at the deleted switch |
| 744 | main.log.warn( str(device) ) |
| 745 | result = main.FALSE |
| 746 | if device and device['available'] == True: |
| 747 | result = main.TRUE |
| 748 | utilities.assert_equals(expect=main.TRUE,actual=result, |
| 749 | onpass="add switch succesful", |
| 750 | onfail="Failed to add switch?") |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 751 | |
| 752 | def CASE13 (self, main) : |
| 753 | ''' |
| 754 | Clean up |
| 755 | ''' |
| 756 | import os |
| 757 | import time |
| 758 | description = "Test Cleanup" |
| 759 | main.log.report(description) |
| 760 | main.case(description) |
| 761 | main.step("Killing tcpdumps") |
| 762 | main.Mininet2.stop_tcpdump() |
| 763 | |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 764 | main.step("Checking ONOS Logs for errors") |
| 765 | print "Checking logs for errors on ONOS1:" |
| 766 | print main.ONOSbench.check_logs(ONOS1_ip) |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 767 | main.step("Copying MN pcap and ONOS log files to test station") |
| 768 | testname = main.TEST |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 769 | teststation_user = main.params['TESTONUSER'] |
| 770 | teststation_IP = main.params['TESTONIP'] |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 771 | #NOTE: MN Pcap file is being saved to ~/packet_captures |
| 772 | # scp this file as MN and TestON aren't necessarily the same vm |
| 773 | #FIXME: scp |
| 774 | #####mn files |
| 775 | #TODO: Load these from params |
| 776 | #NOTE: must end in / |
| 777 | log_folder = "/opt/onos/log/" |
| 778 | log_files = ["karaf.log", "karaf.log.1"] |
| 779 | #NOTE: must end in / |
| 780 | dst_dir = "~/packet_captures/" |
| 781 | for f in log_files: |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 782 | main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+ |
| 783 | teststation_user +"@"+teststation_IP+":"+\ |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 784 | dst_dir + str(testname) + "-ONOS1-"+f ) |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 785 | main.ONOSbench.handle.expect("\$") |
| 786 | print main.ONOSbench.handle.before |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 787 | |
| 788 | #std*.log's |
| 789 | #NOTE: must end in / |
| 790 | log_folder = "/opt/onos/var/" |
| 791 | log_files = ["stderr.log", "stdout.log"] |
| 792 | #NOTE: must end in / |
| 793 | dst_dir = "~/packet_captures/" |
| 794 | for f in log_files: |
Jon Hall | 94fd047 | 2014-12-08 11:52:42 -0800 | [diff] [blame] | 795 | main.ONOSbench.handle.sendline( "scp sdn@"+ONOS1_ip+":"+log_folder+f+" "+ |
| 796 | teststation_user +"@"+teststation_IP+":"+\ |
Jon Hall | 73cf9cc | 2014-11-20 22:28:38 -0800 | [diff] [blame] | 797 | dst_dir + str(testname) + "-ONOS1-"+f ) |
| 798 | |
| 799 | |
| 800 | #sleep so scp can finish |
| 801 | time.sleep(10) |
| 802 | main.step("Packing and rotating pcap archives") |
| 803 | os.system("~/TestON/dependencies/rotate.sh "+ str(testname)) |
| 804 | |
| 805 | |
| 806 | #TODO: actually check something here |
| 807 | utilities.assert_equals(expect=main.TRUE, actual=main.TRUE, |
| 808 | onpass="Test cleanup successful", |
| 809 | onfail="Test cleanup NOT successful") |