andrewonlab | 4a5c447 | 2014-10-09 12:11:41 -0400 | [diff] [blame] | 1 | |
| 2 | #Testing the basic functionality of ONOS Next |
| 3 | #For sanity and driver functionality excercises only. |
| 4 | |
andrewonlab | 7735d85 | 2014-10-09 13:02:47 -0400 | [diff] [blame] | 5 | import time |
| 6 | import sys |
| 7 | import os |
andrewonlab | 2d35516 | 2014-10-10 00:43:59 -0400 | [diff] [blame] | 8 | import re |
andrewonlab | 7735d85 | 2014-10-09 13:02:47 -0400 | [diff] [blame] | 9 | |
andrewonlab | 4a5c447 | 2014-10-09 12:11:41 -0400 | [diff] [blame] | 10 | class ONOSNextTest: |
| 11 | def __init__(self): |
| 12 | self.default = '' |
| 13 | |
| 14 | def CASE1(self, main): |
| 15 | ''' |
| 16 | Startup sequence: |
| 17 | git pull |
| 18 | mvn clean install |
| 19 | onos-package |
| 20 | cell <name> |
| 21 | onos-verify-cell |
| 22 | onos-install -f |
| 23 | onos-wait-for-start |
| 24 | ''' |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 25 | import time |
| 26 | |
andrewonlab | 084307e | 2014-10-09 14:06:09 -0400 | [diff] [blame] | 27 | cell_name = main.params['ENV']['cellName'] |
andrewonlab | ec9c321 | 2014-10-17 20:02:53 -0400 | [diff] [blame^] | 28 | |
andrewonlab | 084307e | 2014-10-09 14:06:09 -0400 | [diff] [blame] | 29 | ONOS1_ip = main.params['CTRL']['ip1'] |
andrew@onlab.us | 59e8f69 | 2014-10-09 21:41:48 -0400 | [diff] [blame] | 30 | ONOS1_port = main.params['CTRL']['port1'] |
Jon Hall | ea7818b | 2014-10-09 14:30:59 -0400 | [diff] [blame] | 31 | |
andrewonlab | ec9c321 | 2014-10-17 20:02:53 -0400 | [diff] [blame^] | 32 | git_pull_trigger = main.params['GIT']['autoPull'] |
| 33 | git_checkout_branch = main.params['GIT']['branch'] |
| 34 | |
andrewonlab | 8d0d7d7 | 2014-10-09 16:33:15 -0400 | [diff] [blame] | 35 | main.case("Setting up test environment") |
Jon Hall | 3be576e | 2014-10-09 22:23:03 -0400 | [diff] [blame] | 36 | |
andrewonlab | 9428209 | 2014-10-10 13:00:11 -0400 | [diff] [blame] | 37 | main.step("Creating cell file") |
andrewonlab | 2c010fb | 2014-10-10 18:08:59 -0400 | [diff] [blame] | 38 | #params: (bench ip, cell name, mininet ip, *onos ips) |
andrewonlab | 9428209 | 2014-10-10 13:00:11 -0400 | [diff] [blame] | 39 | cell_file_result = main.ONOSbench.create_cell_file( |
| 40 | "10.128.20.10", "temp_cell_2", "10.128.10.90", |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 41 | "onos-core-trivial,onos-app-fwd", |
| 42 | "10.128.20.11") |
andrewonlab | 9428209 | 2014-10-10 13:00:11 -0400 | [diff] [blame] | 43 | |
andrewonlab | 8d0d7d7 | 2014-10-09 16:33:15 -0400 | [diff] [blame] | 44 | main.step("Applying cell variable to environment") |
andrewonlab | ec9c321 | 2014-10-17 20:02:53 -0400 | [diff] [blame^] | 45 | #cell_result = main.ONOSbench.set_cell(cell_name) |
| 46 | cell_result = main.ONOSbench.set_cell("temp_cell_2") |
andrewonlab | c03bf6c | 2014-10-09 14:56:18 -0400 | [diff] [blame] | 47 | verify_result = main.ONOSbench.verify_cell() |
andrewonlab | ec9c321 | 2014-10-17 20:02:53 -0400 | [diff] [blame^] | 48 | |
| 49 | if git_pull_trigger == 'on': |
| 50 | main.step("Git checkout and pull master") |
| 51 | main.ONOSbench.git_checkout(git_checkout_branch) |
| 52 | git_pull_result = main.ONOSbench.git_pull() |
| 53 | else: |
| 54 | main.log.info("Git checkout and pull skipped by config") |
| 55 | git_pull_result = main.TRUE |
| 56 | |
Jon Hall | 77f53ce | 2014-10-13 18:02:06 -0400 | [diff] [blame] | 57 | main.step("Using mvn clean & install") |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 58 | #clean_install_result = main.ONOSbench.clean_install() |
| 59 | clean_install_result = main.TRUE |
Jon Hall | 77f53ce | 2014-10-13 18:02:06 -0400 | [diff] [blame] | 60 | |
| 61 | main.step("Creating ONOS package") |
| 62 | package_result = main.ONOSbench.onos_package() |
| 63 | |
andrewonlab | 2fe32ad | 2014-10-09 16:36:40 -0400 | [diff] [blame] | 64 | main.step("Installing ONOS package") |
Jon Hall | 7993bfc | 2014-10-09 16:30:14 -0400 | [diff] [blame] | 65 | onos_install_result = main.ONOSbench.onos_install() |
| 66 | onos1_isup = main.ONOSbench.isup() |
andrewonlab | 8d0d7d7 | 2014-10-09 16:33:15 -0400 | [diff] [blame] | 67 | |
| 68 | main.step("Starting ONOS service") |
| 69 | start_result = main.ONOSbench.onos_start(ONOS1_ip) |
| 70 | |
Jon Hall | 7993bfc | 2014-10-09 16:30:14 -0400 | [diff] [blame] | 71 | case1_result = (clean_install_result and package_result and\ |
andrewonlab | 2b30bd3 | 2014-10-09 16:48:55 -0400 | [diff] [blame] | 72 | cell_result and verify_result and onos_install_result and\ |
andrewonlab | f103e9f | 2014-10-10 00:59:07 -0400 | [diff] [blame] | 73 | onos1_isup and start_result ) |
andrewonlab | 0748d2a | 2014-10-09 13:24:17 -0400 | [diff] [blame] | 74 | utilities.assert_equals(expect=main.TRUE, actual=case1_result, |
| 75 | onpass="Test startup successful", |
| 76 | onfail="Test startup NOT successful") |
andrewonlab | 4a5c447 | 2014-10-09 12:11:41 -0400 | [diff] [blame] | 77 | |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 78 | time.sleep(10) |
| 79 | |
andrewonlab | 05e362f | 2014-10-10 00:40:57 -0400 | [diff] [blame] | 80 | def CASE11(self, main): |
andrewonlab | 2b30bd3 | 2014-10-09 16:48:55 -0400 | [diff] [blame] | 81 | ''' |
| 82 | Cleanup sequence: |
| 83 | onos-service <node_ip> stop |
| 84 | onos-uninstall |
| 85 | |
| 86 | TODO: Define rest of cleanup |
| 87 | |
| 88 | ''' |
| 89 | |
| 90 | ONOS1_ip = main.params['CTRL']['ip1'] |
| 91 | |
| 92 | main.case("Cleaning up test environment") |
| 93 | |
andrewonlab | 057c06a | 2014-10-09 17:19:21 -0400 | [diff] [blame] | 94 | main.step("Testing ONOS kill function") |
| 95 | kill_result = main.ONOSbench.onos_kill(ONOS1_ip) |
| 96 | |
andrewonlab | 2b30bd3 | 2014-10-09 16:48:55 -0400 | [diff] [blame] | 97 | main.step("Stopping ONOS service") |
| 98 | stop_result = main.ONOSbench.onos_stop(ONOS1_ip) |
| 99 | |
| 100 | main.step("Uninstalling ONOS service") |
andrewonlab | 057c06a | 2014-10-09 17:19:21 -0400 | [diff] [blame] | 101 | uninstall_result = main.ONOSbench.onos_uninstall() |
andrewonlab | 4a5c447 | 2014-10-09 12:11:41 -0400 | [diff] [blame] | 102 | |
andrewonlab | 05e362f | 2014-10-10 00:40:57 -0400 | [diff] [blame] | 103 | def CASE3(self, main): |
| 104 | ''' |
| 105 | Test 'onos' command and its functionality in driver |
| 106 | ''' |
| 107 | |
| 108 | ONOS1_ip = main.params['CTRL']['ip1'] |
andrewonlab | 05e362f | 2014-10-10 00:40:57 -0400 | [diff] [blame] | 109 | |
| 110 | main.case("Testing 'onos' command") |
| 111 | |
andrewonlab | f103e9f | 2014-10-10 00:59:07 -0400 | [diff] [blame] | 112 | main.step("Sending command 'onos -w <onos-ip> system:name'") |
| 113 | cmdstr1 = "system:name" |
| 114 | cmd_result1 = main.ONOSbench.onos_cli(ONOS1_ip, cmdstr1) |
| 115 | main.log.info("onos command returned: "+cmd_result1) |
| 116 | |
| 117 | main.step("Sending command 'onos -w <onos-ip> onos:topology'") |
| 118 | cmdstr2 = "onos:topology" |
| 119 | cmd_result2 = main.ONOSbench.onos_cli(ONOS1_ip, cmdstr2) |
| 120 | main.log.info("onos command returned: "+cmd_result2) |
Jon Hall | 77f53ce | 2014-10-13 18:02:06 -0400 | [diff] [blame] | 121 | |
| 122 | main.step("Testing check_status") |
| 123 | check_status_results = main.ONOSbench.check_status(ONOS1_ip, 4, 6) |
| 124 | main.log.info("Results of check_status " + str(check_status_results)) |
andrewonlab | f103e9f | 2014-10-10 00:59:07 -0400 | [diff] [blame] | 125 | |
andrewonlab | 2c010fb | 2014-10-10 18:08:59 -0400 | [diff] [blame] | 126 | main.step("Sending command 'onos -w <onos-ip> bundle:list'") |
| 127 | cmdstr3 = "bundle:list" |
| 128 | cmd_result3 = main.ONOSbench.onos_cli(ONOS1_ip, cmdstr3) |
| 129 | main.log.info("onos command returned: "+cmd_result3) |
Jon Hall | 77f53ce | 2014-10-13 18:02:06 -0400 | [diff] [blame] | 130 | case3_result = (cmd_result1 and cmd_result2 and\ |
| 131 | check_status_results and cmd_result3 ) |
| 132 | utilities.assert_equals(expect=main.TRUE, actual=case3_result, |
| 133 | onpass="Test case 3 successful", |
| 134 | onfail="Test case 3 NOT successful") |
andrewonlab | 2c010fb | 2014-10-10 18:08:59 -0400 | [diff] [blame] | 135 | |
andrewonlab | f103e9f | 2014-10-10 00:59:07 -0400 | [diff] [blame] | 136 | def CASE4(self, main): |
Jon Hall | 3173533 | 2014-10-10 15:27:43 -0400 | [diff] [blame] | 137 | import re |
| 138 | import time |
Jon Hall | 77f53ce | 2014-10-13 18:02:06 -0400 | [diff] [blame] | 139 | main.case("Pingall Test(No intents are added)") |
andrewonlab | f103e9f | 2014-10-10 00:59:07 -0400 | [diff] [blame] | 140 | main.step("Assigning switches to controllers") |
Jon Hall | 77f53ce | 2014-10-13 18:02:06 -0400 | [diff] [blame] | 141 | for i in range(1,5): #1 to (num of switches +1) |
andrewonlab | f103e9f | 2014-10-10 00:59:07 -0400 | [diff] [blame] | 142 | main.Mininet1.assign_sw_controller(sw=str(i), |
| 143 | ip1=ONOS1_ip, port1=ONOS1_port) |
| 144 | switch_mastership = main.TRUE |
Jon Hall | 77f53ce | 2014-10-13 18:02:06 -0400 | [diff] [blame] | 145 | for i in range (1,5): |
andrewonlab | f103e9f | 2014-10-10 00:59:07 -0400 | [diff] [blame] | 146 | response = main.Mininet1.get_sw_controller("s"+str(i)) |
| 147 | print("Response is " + str(response)) |
| 148 | if re.search("tcp:"+ONOS1_ip,response): |
| 149 | switch_mastership = switch_mastership and main.TRUE |
| 150 | else: |
| 151 | switch_mastership = main.FALSE |
| 152 | |
| 153 | #REACTIVE FWD test |
| 154 | main.step("Pingall") |
| 155 | ping_result = main.FALSE |
| 156 | while ping_result == main.FALSE: |
| 157 | time1 = time.time() |
| 158 | ping_result = main.Mininet1.pingall() |
| 159 | time2 = time.time() |
| 160 | print "Time for pingall: %2f seconds" % (time2 - time1) |
| 161 | |
| 162 | case4_result = switch_mastership and ping_result |
| 163 | utilities.assert_equals(expect=main.TRUE, actual=case4_result, |
Jon Hall | 3173533 | 2014-10-10 15:27:43 -0400 | [diff] [blame] | 164 | onpass="Pingall Test successful", |
| 165 | onfail="Pingall Test NOT successful") |
andrewonlab | f103e9f | 2014-10-10 00:59:07 -0400 | [diff] [blame] | 166 | |
andrewonlab | c7345f2 | 2014-10-13 14:12:59 -0400 | [diff] [blame] | 167 | def CASE5(self, main): |
| 168 | ''' |
| 169 | Test the ONOS-cli functionality |
andrewonlab | 7e4d2d3 | 2014-10-15 13:23:21 -0400 | [diff] [blame] | 170 | |
andrewonlab | 3e15ead | 2014-10-15 14:21:34 -0400 | [diff] [blame] | 171 | Below are demonstrations of what the |
| 172 | ONOS cli driver functions can be used for. |
andrewonlab | c7345f2 | 2014-10-13 14:12:59 -0400 | [diff] [blame] | 173 | ''' |
andrewonlab | 86dc308 | 2014-10-13 18:18:38 -0400 | [diff] [blame] | 174 | import time |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 175 | import json |
| 176 | |
andrewonlab | c7345f2 | 2014-10-13 14:12:59 -0400 | [diff] [blame] | 177 | cell_name = main.params['ENV']['cellName'] |
| 178 | ONOS1_ip = main.params['CTRL']['ip1'] |
| 179 | |
| 180 | main.case("Testing the ONOS-cli") |
| 181 | |
| 182 | main.step("Set cell for ONOS-cli environment") |
| 183 | main.ONOScli.set_cell(cell_name) |
| 184 | |
| 185 | main.step("Start ONOS-cli") |
andrewonlab | 86dc308 | 2014-10-13 18:18:38 -0400 | [diff] [blame] | 186 | main.ONOScli.start_onos_cli(ONOS1_ip) |
andrewonlab | c7345f2 | 2014-10-13 14:12:59 -0400 | [diff] [blame] | 187 | |
| 188 | main.step("issue command: onos:topology") |
andrewonlab | 7c21157 | 2014-10-15 16:45:20 -0400 | [diff] [blame] | 189 | topology_obj = main.ONOScli.topology() |
andrewonlab | 86dc308 | 2014-10-13 18:18:38 -0400 | [diff] [blame] | 190 | |
| 191 | main.step("issue various feature:install <str> commands") |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 192 | #main.ONOScli.feature_install("onos-app-fwd") |
| 193 | #main.ONOScli.feature_install("onos-rest") |
andrewonlab | 86dc308 | 2014-10-13 18:18:38 -0400 | [diff] [blame] | 194 | |
| 195 | main.step("Add a bad node") |
| 196 | node_result = main.ONOScli.add_node("111", "10.128.20.") |
| 197 | if node_result == main.TRUE: |
| 198 | main.log.info("Node successfully added") |
| 199 | |
| 200 | main.step("Add a correct node") |
| 201 | node_result = main.ONOScli.add_node("111", "10.128.20.12") |
| 202 | |
andrewonlab | 7e4d2d3 | 2014-10-15 13:23:21 -0400 | [diff] [blame] | 203 | main.step("Assign switches and list devices") |
andrewonlab | 86dc308 | 2014-10-13 18:18:38 -0400 | [diff] [blame] | 204 | for i in range(1,8): |
andrewonlab | f80585f | 2014-10-13 19:00:11 -0400 | [diff] [blame] | 205 | main.Mininet2.handle.sendline("sh ovs-vsctl set-controller s"+str(i)+ |
andrewonlab | 86dc308 | 2014-10-13 18:18:38 -0400 | [diff] [blame] | 206 | " tcp:10.128.20.11") |
andrewonlab | f80585f | 2014-10-13 19:00:11 -0400 | [diff] [blame] | 207 | main.Mininet2.handle.expect("mininet>") |
andrewonlab | 86dc308 | 2014-10-13 18:18:38 -0400 | [diff] [blame] | 208 | #Need to sleep to allow switch add processing |
andrewonlab | 7e4d2d3 | 2014-10-15 13:23:21 -0400 | [diff] [blame] | 209 | time.sleep(5) |
andrewonlab | 9a50dfe | 2014-10-17 17:22:31 -0400 | [diff] [blame] | 210 | list_result = main.ONOScli.devices(json_format=False) |
andrewonlab | 86dc308 | 2014-10-13 18:18:38 -0400 | [diff] [blame] | 211 | main.log.info(list_result) |
| 212 | |
andrewonlab | 7e4d2d3 | 2014-10-15 13:23:21 -0400 | [diff] [blame] | 213 | main.step("Get all devices id") |
| 214 | devices_id_list = main.ONOScli.get_all_devices_id() |
| 215 | main.log.info(devices_id_list) |
andrewonlab | c7345f2 | 2014-10-13 14:12:59 -0400 | [diff] [blame] | 216 | |
andrewonlab | 3e15ead | 2014-10-15 14:21:34 -0400 | [diff] [blame] | 217 | main.step("Get path and cost between device 1 and 7") |
| 218 | (path, cost) = main.ONOScli.paths(devices_id_list[0], devices_id_list[6]) |
| 219 | main.log.info("Path: "+str(path)) |
| 220 | main.log.info("Cost: "+str(cost)) |
andrewonlab | 7c21157 | 2014-10-15 16:45:20 -0400 | [diff] [blame] | 221 | |
| 222 | main.step("Get nodes currently visible") |
| 223 | nodes_str = main.ONOScli.nodes() |
| 224 | main.log.info(nodes_str) |
| 225 | |
| 226 | main.step("Get all nodes id's") |
| 227 | node_id_list = main.ONOScli.get_all_nodes_id() |
| 228 | main.log.info(node_id_list) |
| 229 | |
| 230 | main.step("Set device "+str(devices_id_list[0])+" to role: standby") |
| 231 | device_role_result = main.ONOScli.device_role( |
| 232 | devices_id_list[0], node_id_list[0], "standby") |
| 233 | if device_role_result == main.TRUE: |
| 234 | main.log.report("Device role successfully set") |
| 235 | |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 236 | main.step("Revert device role to master") |
| 237 | device_role = main.ONOScli.device_role( |
| 238 | devices_id_list[0], node_id_list[0], "master") |
| 239 | |
andrewonlab | 7c21157 | 2014-10-15 16:45:20 -0400 | [diff] [blame] | 240 | main.step("Check devices / role again") |
andrewonlab | 9a50dfe | 2014-10-17 17:22:31 -0400 | [diff] [blame] | 241 | dev_result = main.ONOScli.devices(json_format=False) |
andrewonlab | 7c21157 | 2014-10-15 16:45:20 -0400 | [diff] [blame] | 242 | main.log.info(dev_result) |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 243 | |
| 244 | #Sample steps to push intents *********** |
| 245 | # * Obtain host id in ONOS format |
| 246 | # * Push intents |
| 247 | main.step("Get list of hosts from Mininet") |
| 248 | host_list = main.Mininet2.get_hosts() |
| 249 | main.log.info(host_list) |
| 250 | |
| 251 | main.step("Get host list in ONOS format") |
| 252 | host_onos_list = main.ONOScli.get_hosts_id(host_list) |
| 253 | main.log.info(host_onos_list) |
| 254 | |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 255 | time.sleep(5) |
| 256 | |
andrewonlab | 9a50dfe | 2014-10-17 17:22:31 -0400 | [diff] [blame] | 257 | #We must use ping from hosts we want to add intents from |
| 258 | #to make the hosts talk |
| 259 | #main.Mininet2.handle.sendline("\r") |
| 260 | #main.Mininet2.handle.sendline("h4 ping 10.1.1.1 -c 1 -W 1") |
| 261 | #time.sleep(3) |
| 262 | #main.Mininet2.handle.sendline("h5 ping 10.1.1.1 -c 1 -W 1") |
| 263 | #time.sleep(5) |
| 264 | |
| 265 | main.ONOScli.feature_install("onos-app-fwd") |
| 266 | |
| 267 | main.Mininet2.pingall() |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 268 | |
| 269 | time.sleep(5) |
| 270 | |
| 271 | main.step("Get hosts") |
| 272 | main.ONOScli.handle.sendline("hosts") |
| 273 | main.ONOScli.handle.expect("onos>") |
| 274 | hosts = main.ONOScli.handle.before |
| 275 | main.log.info(hosts) |
| 276 | |
andrewonlab | 9a50dfe | 2014-10-17 17:22:31 -0400 | [diff] [blame] | 277 | main.step("Install host-to-host-intents") |
| 278 | intent_install1 = main.ONOScli.add_host_intent( |
| 279 | host_onos_list[0], host_onos_list[1]) |
| 280 | intent_install2 = main.ONOScli.add_host_intent( |
| 281 | host_onos_list[2], host_onos_list[3]) |
| 282 | intent_install3 = main.ONOScli.add_host_intent( |
| 283 | host_onos_list[4], host_onos_list[5]) |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 284 | |
andrewonlab | 9a50dfe | 2014-10-17 17:22:31 -0400 | [diff] [blame] | 285 | main.log.info(intent_install1) |
| 286 | main.log.info(intent_install2) |
| 287 | main.log.info(intent_install3) |
andrewonlab | e674534 | 2014-10-17 14:29:13 -0400 | [diff] [blame] | 288 | |
| 289 | main.step("Get intents installed on ONOS") |
| 290 | get_intent_result = main.ONOScli.intents() |
| 291 | main.log.info(get_intent_result) |
| 292 | #**************************************** |
| 293 | |
andrewonlab | 9a50dfe | 2014-10-17 17:22:31 -0400 | [diff] [blame] | 294 | #Sample steps to delete intents ******** |
| 295 | main.step("Get all intent id's") |
| 296 | intent_id = main.ONOScli.get_all_intents_id() |
| 297 | main.log.info(intent_id) |
| 298 | |
| 299 | main.step("Remove specified intent id: "+str(intent_id[0])) |
| 300 | intent_result = main.ONOScli.remove_intent(intent_id[0]) |
| 301 | main.log.info(intent_result) |
| 302 | |
| 303 | main.step("Check installed intents again") |
| 304 | get_intent_result = main.ONOScli.intents() |
| 305 | main.log.info(get_intent_result) |
| 306 | #*************************************** |
| 307 | |
andrewonlab | 4dbb4d8 | 2014-10-17 18:22:31 -0400 | [diff] [blame] | 308 | #Sample steps to add point-to-point intents* |
| 309 | main.step("Add point-to-point intents") |
| 310 | ptp_intent_result = main.ONOScli.add_point_intent( |
| 311 | devices_id_list[0], 1, devices_id_list[1], 2) |
| 312 | if ptp_intent_result == main.TRUE: |
| 313 | get_intent_result = main.ONOScli.intents() |
| 314 | main.log.info("Point to point intent install successful") |
| 315 | main.log.info(get_intent_result) |
| 316 | #******************************************* |
andrewonlab | 7c21157 | 2014-10-15 16:45:20 -0400 | [diff] [blame] | 317 | |
andrewonlab | 7e4d2d3 | 2014-10-15 13:23:21 -0400 | [diff] [blame] | 318 | ###### |
| 319 | #jhall@onlab.us |
| 320 | #andrew@onlab.us |
| 321 | ###### |