admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 1 | |
| 2 | class RCOnosScale4nodes: |
| 3 | |
| 4 | def __init__(self) : |
| 5 | self.default = '' |
| 6 | |
| 7 | def CASE1(self,main) : |
| 8 | ''' |
| 9 | First case is to simply check if ONOS, ZK, and RamCloud are all running properly. |
| 10 | If ONOS if not running properly, it will restart ONOS once before continuing. |
| 11 | It will then check if the ONOS has a view of all the switches and links as defined in the params file. |
| 12 | The test will only pass if ONOS is running properly, and has a full view of all topology elements. |
| 13 | ''' |
| 14 | import time |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 15 | main.case("Initial setup") |
| 16 | main.step("stopping ONOS") |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 17 | main.ONOS1.stop() |
| 18 | main.ONOS2.stop() |
| 19 | main.ONOS3.stop() |
| 20 | main.ONOS4.stop() |
admin | aef0055 | 2014-05-08 09:18:36 -0700 | [diff] [blame] | 21 | main.RamCloud1.stop_coor() |
| 22 | main.RamCloud1.stop_serv() |
| 23 | main.RamCloud2.stop_serv() |
| 24 | main.RamCloud3.stop_serv() |
| 25 | main.RamCloud4.stop_serv() |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 26 | main.step("Start tcpdump on mn") |
| 27 | main.Mininet1.start_tcpdump(main.params['tcpdump']['filename'], intf = main.params['tcpdump']['intf'], port = main.params['tcpdump']['port']) |
| 28 | main.step("Starting ONOS") |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 29 | main.Zookeeper1.start() |
| 30 | main.Zookeeper2.start() |
| 31 | main.Zookeeper3.start() |
| 32 | main.Zookeeper4.start() |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 33 | time.sleep(10) |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 34 | main.RamCloud1.del_db() |
| 35 | main.RamCloud2.del_db() |
| 36 | main.RamCloud3.del_db() |
| 37 | main.RamCloud4.del_db() |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 38 | |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 39 | main.RamCloud1.start_coor() |
Jon Hall | 76f2c46 | 2014-04-17 11:37:15 -0700 | [diff] [blame] | 40 | time.sleep(10) |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 41 | main.RamCloud1.start_serv() |
| 42 | main.RamCloud2.start_serv() |
| 43 | main.RamCloud3.start_serv() |
| 44 | main.RamCloud4.start_serv() |
| 45 | time.sleep(20) |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 46 | main.ONOS1.start() |
| 47 | time.sleep(10) |
| 48 | main.ONOS2.start() |
| 49 | main.ONOS3.start() |
| 50 | main.ONOS4.start() |
| 51 | main.ONOS1.start_rest() |
| 52 | time.sleep(5) |
| 53 | test= main.ONOS1.rest_status() |
| 54 | if test == main.FALSE: |
| 55 | main.ONOS1.start_rest() |
| 56 | main.ONOS1.get_version() |
| 57 | main.log.report("Startup check Zookeeper1, RamCloud1, and ONOS1 connections") |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 58 | main.step("Testing startup Zookeeper") |
| 59 | data = main.Zookeeper1.isup() |
| 60 | utilities.assert_equals(expect=main.TRUE,actual=data,onpass="Zookeeper is up!",onfail="Zookeeper is down...") |
| 61 | main.step("Testing startup RamCloud") |
admin | c6cfc1c | 2014-04-21 13:55:21 -0700 | [diff] [blame] | 62 | data = main.RamCloud1.status_serv() |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 63 | if data == main.FALSE: |
| 64 | main.RamCloud1.stop_coor() |
| 65 | main.RamCloud1.stop_serv() |
| 66 | main.RamCloud2.stop_serv() |
| 67 | main.RamCloud3.stop_serv() |
| 68 | main.RamCloud4.stop_serv() |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 69 | main.RamCloud1.del_db() |
| 70 | main.RamCloud2.del_db() |
| 71 | main.RamCloud3.del_db() |
| 72 | main.RamCloud4.del_db() |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 73 | |
| 74 | time.sleep(5) |
| 75 | |
| 76 | main.RamCloud1.start_coor() |
| 77 | main.RamCloud1.start_serv() |
| 78 | main.RamCloud2.start_serv() |
| 79 | main.RamCloud3.start_serv() |
| 80 | main.RamCloud4.start_serv() |
Jon Hall | 76f2c46 | 2014-04-17 11:37:15 -0700 | [diff] [blame] | 81 | data = main.RamCloud1.isup() |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 82 | utilities.assert_equals(expect=main.TRUE,actual=data,onpass="RamCloud is up!",onfail="RamCloud is down...") |
| 83 | main.step("Testing startup ONOS") |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 84 | time.sleep(10) |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 85 | data = main.ONOS1.isup() |
| 86 | data = data and main.ONOS2.isup() |
| 87 | data = data and main.ONOS3.isup() |
| 88 | data = data and main.ONOS4.isup() |
| 89 | if data == main.FALSE: |
| 90 | main.log.report("Something is funny... restarting ONOS") |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 91 | time.sleep(10) |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 92 | data = main.ONOS1.isup() |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 93 | data = data and main.ONOS2.isup() |
| 94 | data = data and main.ONOS3.isup() |
| 95 | data = data and main.ONOS4.isup() |
| 96 | |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 97 | utilities.assert_equals(expect=main.TRUE,actual=data,onpass="ONOS is up and running!",onfail="ONOS didn't start...") |
| 98 | |
| 99 | def CASE2(self,main) : |
| 100 | ''' |
| 101 | Second case is to time the convergence time of a topology for ONOS. |
| 102 | It shuts down the ONOS, drops keyspace, starts ONOS... |
| 103 | Then it points all the mininet switches at the ONOS node and times how long it take for the ONOS rest call to reflect the correct count of switches and links. |
| 104 | ''' |
| 105 | import time |
| 106 | main.log.report("Time convergence for switches -> single ONOS node in cluster") |
| 107 | main.case("Timing Onos Convergence for switch -> a single ONOS node in the cluster") |
| 108 | main.step("Bringing ONOS down...") |
| 109 | main.log.info("all switch no controllers") |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 110 | for i in range(1,int(main.params['NR_Switches'])+1): |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 111 | main.Mininet1.delete_sw_controller("s"+str(i)) |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 112 | main.log.info("bringing ONOS down") |
| 113 | main.ONOS1.stop() |
| 114 | main.ONOS2.stop() |
| 115 | main.ONOS3.stop() |
| 116 | main.ONOS4.stop() |
Jon Hall | 76f2c46 | 2014-04-17 11:37:15 -0700 | [diff] [blame] | 117 | main.RamCloud1.stop_coor() |
| 118 | main.RamCloud1.stop_serv() |
| 119 | main.RamCloud2.stop_serv() |
| 120 | main.RamCloud3.stop_serv() |
| 121 | main.RamCloud4.stop_serv() |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 122 | main.Zookeeper1.start() |
| 123 | main.Zookeeper2.start() |
| 124 | main.Zookeeper3.start() |
| 125 | main.Zookeeper4.start() |
| 126 | time.sleep(4) |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 127 | main.RamCloud1.del_db() |
| 128 | main.RamCloud2.del_db() |
| 129 | main.RamCloud3.del_db() |
| 130 | main.RamCloud4.del_db() |
Jon Hall | 76f2c46 | 2014-04-17 11:37:15 -0700 | [diff] [blame] | 131 | |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 132 | time.sleep(5) |
Jon Hall | 76f2c46 | 2014-04-17 11:37:15 -0700 | [diff] [blame] | 133 | |
| 134 | main.RamCloud1.start_coor() |
| 135 | main.RamCloud1.start_serv() |
| 136 | main.RamCloud2.start_serv() |
| 137 | main.RamCloud3.start_serv() |
| 138 | main.RamCloud4.start_serv() |
| 139 | |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 140 | main.log.info("Bringing ONOS up") |
| 141 | main.ONOS1.start() |
| 142 | time.sleep(5) |
| 143 | main.ONOS2.start() |
| 144 | main.ONOS3.start() |
| 145 | main.ONOS4.start() |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 146 | time.sleep(10) |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 147 | data = main.ONOS1.isup() |
| 148 | data = data and main.ONOS2.isup() |
| 149 | data = data and main.ONOS3.isup() |
| 150 | data = data and main.ONOS4.isup() |
| 151 | if data == main.FALSE: |
| 152 | main.log.report("Something is funny... restarting ONOS") |
| 153 | time.sleep(10) |
| 154 | data = main.ONOS1.isup() |
| 155 | data = data and main.ONOS2.isup() |
| 156 | data = data and main.ONOS3.isup() |
| 157 | data = data and main.ONOS4.isup() |
| 158 | if data == main.FALSE: |
| 159 | main.log.report("Something is funny... restarting ONOS") |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 160 | main.ONOS1.check_status(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 161 | |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 162 | main.log.info("Pointing the Switches at ONE controller... then BEGIN time") |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 163 | for i in range(1,int(main.params['NR_Switches'])+1): |
| 164 | main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1']) |
| 165 | |
| 166 | |
| 167 | |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 168 | t1 = time.time() |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 169 | for i in range(15) : |
| 170 | result = main.ONOS1.check_status(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 171 | if result == 1 : |
| 172 | break |
| 173 | else : |
| 174 | time.sleep(1) |
| 175 | t2 = time.time() |
| 176 | conv_time = t2 - t1 |
| 177 | main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 178 | if result == 1 : |
| 179 | main.log.report( "Convergence time of : %f seconds" % conv_time ) |
| 180 | if float(conv_time) < float(main.params['TargetTime']) : |
| 181 | test=main.TRUE |
| 182 | main.log.info("Time is less then supplied target time") |
| 183 | else: |
| 184 | test=main.FALSE |
| 185 | main.log.info("Time is greater then supplied target time") |
| 186 | else : |
| 187 | main.log.report( "ONOS did NOT converge over : %f seconds" % conv_time ) |
| 188 | test=main.FALSE |
| 189 | |
| 190 | utilities.assert_equals(expect=main.TRUE,actual=test) |
| 191 | |
| 192 | def CASE3(self,main) : |
| 193 | ''' |
| 194 | Second case is to time the convergence time of a topology for ONOS. |
| 195 | It shuts down the ONOS, drops keyspace, starts ONOS... |
| 196 | Then it points all the mininet switches at all ONOS nodes and times how long it take for the ONOS rest call to reflect the correct count of switches and links. |
| 197 | ''' |
| 198 | import time |
| 199 | main.log.report("Time convergence for switches -> all ONOS nodes in cluster") |
| 200 | main.case("Timing Onos Convergence for switch -> all ONOS nodes in cluster") |
| 201 | main.step("Bringing ONOS down...") |
| 202 | main.log.info("all switch no controllers") |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 203 | for i in range(1,int(main.params['NR_Switches'])+1): |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 204 | main.Mininet1.delete_sw_controller("s"+str(i)) |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 205 | main.log.info("bringing ONOS down") |
| 206 | main.ONOS1.stop() |
| 207 | main.ONOS2.stop() |
| 208 | main.ONOS3.stop() |
| 209 | main.ONOS4.stop() |
| 210 | #main.log.info("Dropping keyspace...") |
| 211 | #main.ONOS1.drop_keyspace() |
| 212 | time.sleep(5) |
| 213 | main.log.info("Bringing ONOS up") |
| 214 | main.ONOS1.start() |
| 215 | time.sleep(5) |
| 216 | main.ONOS2.start() |
| 217 | main.ONOS2.start_rest() |
| 218 | main.ONOS3.start() |
| 219 | main.ONOS4.start() |
| 220 | main.ONOS1.isup() |
| 221 | main.ONOS2.isup() |
| 222 | main.ONOS3.isup() |
| 223 | main.ONOS4.isup() |
| 224 | main.ONOS1.check_status(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 225 | main.log.info("Pointing the Switches at ALL controllers... then BEGIN time") |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 226 | for i in range(1,int(main.params['NR_Switches'])+1): |
| 227 | main.Mininet1.assign_sw_controller(sw=str(i),count=4,ip1=main.params['CTRL']['ip1'],port1=main.params['CTRL']['port1'],ip2=main.params['CTRL']['ip2'],port2=main.params['CTRL']['port2'],ip3=main.params['CTRL']['ip3'],port3=main.params['CTRL']['port3'],ip4=main.params['CTRL']['ip4'],port4=main.params['CTRL']['port4']) |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 228 | t1 = time.time() |
| 229 | for i in range(15) : |
| 230 | result = main.ONOS1.check_status(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 231 | if result == 1 : |
| 232 | break |
| 233 | else : |
| 234 | time.sleep(1) |
| 235 | t2 = time.time() |
| 236 | conv_time = t2 - t1 |
| 237 | main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 238 | if result == 1 : |
| 239 | main.log.report( "Convergence time of : %f seconds" % conv_time ) |
| 240 | if float(conv_time) < float(main.params['TargetTime']) : |
| 241 | test=main.TRUE |
| 242 | main.log.info("Time is less then supplied target time") |
| 243 | else: |
| 244 | test=main.FALSE |
| 245 | main.log.info("Time is greater then supplied target time") |
| 246 | else : |
| 247 | main.log.report( "ONOS did NOT converge over : %f seconds" % conv_time ) |
| 248 | test=main.FALSE |
| 249 | |
| 250 | utilities.assert_equals(expect=main.TRUE,actual=test) |
| 251 | |
| 252 | def CASE4(self,main) : |
| 253 | ''' |
| 254 | Second case is to time the convergence time of a topology for ONOS. |
| 255 | It shuts down the ONOS, drops keyspace, starts ONOS... |
| 256 | Then it evenly points all mininet switches to all ONOS nodes, but only one node, and times how long it take for the ONOS rest call to reflect the correct count of switches and links. |
| 257 | ''' |
| 258 | import time |
| 259 | main.log.report("Time convergence for switches -> Divide switches equall among all nodes in cluster") |
| 260 | main.case("Timing Onos Convergence for even single controller distribution") |
| 261 | main.step("Bringing ONOS down...") |
| 262 | main.log.info("all switch no controllers") |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 263 | for i in range(1,int(main.params['NR_Switches'])+1): |
| 264 | main.Mininet1.delete_sw_controller("s"+str(i)) |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 265 | main.log.info("bringing ONOS down") |
| 266 | main.ONOS1.stop() |
| 267 | main.ONOS2.stop() |
| 268 | main.ONOS3.stop() |
| 269 | main.ONOS4.stop() |
| 270 | #main.log.info("Dropping keyspace...") |
| 271 | #main.ONOS1.drop_keyspace() |
| 272 | time.sleep(5) |
| 273 | main.log.info("Bringing ONOS up") |
| 274 | main.ONOS1.start() |
| 275 | time.sleep(5) |
| 276 | main.ONOS2.start() |
| 277 | main.ONOS2.start_rest() |
| 278 | main.ONOS3.start() |
| 279 | main.ONOS4.start() |
| 280 | main.ONOS1.isup() |
| 281 | main.ONOS2.isup() |
| 282 | main.ONOS3.isup() |
| 283 | main.ONOS4.isup() |
| 284 | main.ONOS1.check_status(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 285 | |
| 286 | |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 287 | main.log.info("Pointing the Switches to alternating controllers... then BEGIN time") |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 288 | count = 0 |
| 289 | for i in range(1,int(main.params['NR_Switches'])+1): |
Jon Hall | de7bbe8 | 2014-05-23 17:04:31 -0700 | [diff] [blame] | 290 | num = (count % 4)+1 |
Jon Hall | e00b7e4 | 2014-05-23 12:00:33 -0700 | [diff] [blame] | 291 | #num = count % len(controllers) #TODO: check number of controllers in cluster |
| 292 | main.Mininet1.assign_sw_controller(sw=str(i),ip1=main.params['CTRL']['ip'+str(num)],port1=main.params['CTRL']['port'+str(num)]) |
| 293 | count = count + 1 |
| 294 | |
| 295 | |
admin | cb59391 | 2014-04-14 10:34:41 -0700 | [diff] [blame] | 296 | t1 = time.time() |
| 297 | for i in range(15) : |
| 298 | result = main.ONOS1.check_status(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 299 | if result == 1 : |
| 300 | break |
| 301 | else : |
| 302 | time.sleep(1) |
| 303 | t2 = time.time() |
| 304 | conv_time = t2 - t1 |
| 305 | main.ONOS1.check_status_report(main.params['RestIP'],main.params['NR_Switches'],main.params['NR_Links']) |
| 306 | if result == 1 : |
| 307 | main.log.report( "Convergence time of : %f seconds" % conv_time ) |
| 308 | if float(conv_time) < float(main.params['TargetTime']) : |
| 309 | test=main.TRUE |
| 310 | main.log.info("Time is less then supplied target time") |
| 311 | else: |
| 312 | test=main.FALSE |
| 313 | main.log.info("Time is greater then supplied target time") |
| 314 | else : |
| 315 | main.log.report( "ONOS did NOT converge over : %f seconds" % conv_time ) |
| 316 | test=main.FALSE |
| 317 | |
| 318 | utilities.assert_equals(expect=main.TRUE,actual=test) |
| 319 | |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 320 | def CASE66(self, main): |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 321 | main.log.report("Checking ONOS logs for exceptions") |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 322 | count = 0 |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 323 | check1 = main.ONOS1.check_exceptions() |
| 324 | main.log.report("Exceptions in ONOS1 logs: \n" + check1) |
| 325 | check2 = main.ONOS2.check_exceptions() |
| 326 | main.log.report("Exceptions in ONOS2 logs: \n" + check2) |
| 327 | check3 = main.ONOS3.check_exceptions() |
| 328 | main.log.report("Exceptions in ONOS3 logs: \n" + check3) |
| 329 | check4 = main.ONOS4.check_exceptions() |
| 330 | main.log.report("Exceptions in ONOS4 logs: \n" + check4) |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 331 | result = main.TRUE |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 332 | if (check1 or check2 or check3 or check4): |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 333 | result = main.FALSE |
| 334 | count = len(check1.splitlines()) + len(check2.splitlines()) + len(check3.splitlines()) + len(check4.splitlines()) |
| 335 | utilities.assert_equals(expect=main.TRUE,actual=result,onpass="No Exceptions found in the logs",onfail=str(count) + " Exceptions were found in the logs") |
Jon Hall | bd795bf | 2014-06-18 09:46:32 -0700 | [diff] [blame^] | 336 | main.Mininet1.stop_tcpdump() |
admin | 1723f1c | 2014-05-19 16:08:39 -0700 | [diff] [blame] | 337 | |
Jon Hall | 1c4d274 | 2014-05-22 10:57:05 -0700 | [diff] [blame] | 338 | |