Andreas Pantelopoulos | 90f0b10 | 2018-02-01 13:21:45 -0800 | [diff] [blame] | 1 | """ |
| 2 | Copyright 2017 Open Networking Foundation ( ONF ) |
| 3 | |
| 4 | Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>, |
| 5 | the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>, |
| 6 | or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg> |
| 7 | |
| 8 | TestON is free software: you can redistribute it and/or modify |
| 9 | it under the terms of the GNU General Public License as published by |
| 10 | the Free Software Foundation, either version 2 of the License, or |
| 11 | ( at your option ) any later version. |
| 12 | |
| 13 | TestON is distributed in the hope that it will be useful, |
| 14 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 16 | GNU General Public License for more details. |
| 17 | |
| 18 | You should have received a copy of the GNU General Public License |
| 19 | along with TestON. If not, see <http://www.gnu.org/licenses/>. |
| 20 | """ |
| 21 | |
Jon Hall | a604fd4 | 2018-05-04 14:27:27 -0700 | [diff] [blame] | 22 | def setupTest( main, test_idx, onosNodes=-1, ipv4=True, ipv6=True, |
| 23 | external=True, static=False, countFlowsGroups=False ): |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 24 | """ |
| 25 | SRRouting test setup |
| 26 | """ |
| 27 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 28 | import time |
| 29 | |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 30 | skipPackage = False |
| 31 | init = False |
| 32 | if not hasattr( main, 'apps' ): |
| 33 | init = True |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 34 | lib.initTest( main ) |
Jon Hall | a604fd4 | 2018-05-04 14:27:27 -0700 | [diff] [blame] | 35 | if onosNodes < 0: |
| 36 | onosNodes = main.Cluster.numCtrls |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 37 | # Skip onos packaging if the cluster size stays the same |
| 38 | if not init and onosNodes == main.Cluster.numCtrls: |
| 39 | skipPackage = True |
| 40 | |
| 41 | main.internalIpv4Hosts = main.params[ 'TOPO' ][ 'internalIpv4Hosts' ].split( ',' ) |
| 42 | main.internalIpv6Hosts = main.params[ 'TOPO' ][ 'internalIpv6Hosts' ].split( ',' ) |
| 43 | main.externalIpv4Hosts = main.params[ 'TOPO' ][ 'externalIpv4Hosts' ].split( ',' ) |
| 44 | main.externalIpv6Hosts = main.params[ 'TOPO' ][ 'externalIpv6Hosts' ].split( ',' ) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 45 | main.disconnectedIpv4Hosts = [] |
| 46 | main.disconnectedIpv6Hosts = [] |
Jonghwan Hyun | 3759e47 | 2018-05-01 15:40:08 -0700 | [diff] [blame] | 47 | main.disconnectedExternalIpv4Hosts = [ ] |
| 48 | main.disconnectedExternalIpv6Hosts = [ ] |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 49 | main.resultFileName = 'CASE%03d' % test_idx |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 50 | main.Cluster.setRunningNode( onosNodes ) |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 51 | |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 52 | lib.installOnos( main, skipPackage=skipPackage, cliSleep=5, parallel=False ) |
| 53 | |
| 54 | # Load configuration files |
Andreas Pantelopoulos | 1e0665a | 2018-06-05 13:34:59 -0700 | [diff] [blame] | 55 | main.cfgName = 'TEST_CONFIG_ipv4={}_ipv6={}'.format( 1 if ipv4 else 0, |
| 56 | 1 if ipv6 else 0) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 57 | lib.loadJson( main ) |
| 58 | time.sleep( float( main.params[ 'timers' ][ 'loadNetcfgSleep' ] ) ) |
| 59 | lib.loadHost( main ) |
| 60 | |
| 61 | # if static route flag add routes |
| 62 | # these routes are topology specific |
| 63 | if static: |
| 64 | if ipv4: |
| 65 | lib.addStaticOnosRoute( main, "10.0.88.0/24", "10.0.1.1") |
| 66 | if ipv6: |
| 67 | lib.addStaticOnosRoute( main, "2000::8700/120", "2000::101") |
| 68 | if countFlowsGroups: |
| 69 | lib.loadCount( main ) |
| 70 | |
| 71 | if hasattr( main, 'Mininet1' ): |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 72 | # Run the test with Mininet |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 73 | mininet_args = ' --dhcp=1 --routers=1 --ipv6={} --ipv4={}'.format( 1 if ipv6 else 0, |
| 74 | 1 if ipv4 else 0 ) |
| 75 | lib.startMininet( main, main.params[ 'DEPENDENCY' ][ 'topology' ], args=mininet_args ) |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 76 | time.sleep( float( main.params[ "timers" ][ "startMininetSleep" ] ) ) |
| 77 | else: |
| 78 | # Run the test with physical devices |
| 79 | lib.connectToPhysicalNetwork( main, self.switchNames ) |
| 80 | # Check if the devices are up |
| 81 | lib.checkDevices( main, switches=len( self.switchNames ) ) |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 82 | # wait some time for onos to install the rules! |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 83 | time.sleep( float( main.params[ 'timers' ][ 'dhcpSleep' ] ) ) |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 84 | |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 85 | def verifyPingInternal( main, ipv4=True, ipv6=True, disconnected=True ): |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 86 | """ |
| 87 | Verify all connected internal hosts are able to reach each other, |
| 88 | and disconnected internal hosts cannot reach any other internal host |
| 89 | """ |
| 90 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib |
| 91 | # Verify connected hosts |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 92 | if ipv4: |
| 93 | lib.verifyPing( main, |
| 94 | [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ], |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 95 | [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ], |
| 96 | stepMsg="Verify reachability of connected internal IPv4 hosts" ) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 97 | if ipv6: |
| 98 | lib.verifyPing( main, |
| 99 | [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ], |
| 100 | [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ], |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 101 | ipv6=True, acceptableFailed=7, |
| 102 | stepMsg="Verify reachability of connected internal IPv6 hosts" ) |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 103 | # Verify disconnected hosts |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 104 | if disconnected: |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 105 | if main.disconnectedIpv4Hosts: |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 106 | lib.verifyPing( main, main.internalIpv4Hosts, main.disconnectedIpv4Hosts, expect=False, |
| 107 | stepMsg="Verify unreachability of disconnected internal IPv4 hosts" ) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 108 | if main.disconnectedIpv6Hosts: |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 109 | lib.verifyPing( main, main.internalIpv6Hosts, main.disconnectedIpv6Hosts, ipv6=True, expect=False, |
| 110 | stepMsg="Verify unreachability of disconnected internal IPv6 hosts" ) |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 111 | |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 112 | def verifyPingExternal( main, ipv4=True, ipv6=True, disconnected=True ): |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 113 | """ |
| 114 | Verify all connected internal hosts are able to reach external hosts, |
| 115 | and disconnected internal hosts cannot reach any external host |
| 116 | """ |
| 117 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib |
| 118 | # Verify connected hosts |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 119 | if ipv4: |
| 120 | lib.verifyPing( main, |
| 121 | [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ], |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 122 | [ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ], |
You Wang | 54b1d67 | 2018-06-11 16:44:13 -0700 | [diff] [blame^] | 123 | stepMsg="Verify reachability from connected internal IPv4 hosts to external IPv4 hosts", |
| 124 | t3Simple=False ) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 125 | if ipv6: |
| 126 | lib.verifyPing( main, |
| 127 | [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ], |
Jonghwan Hyun | 3759e47 | 2018-05-01 15:40:08 -0700 | [diff] [blame] | 128 | [ h for h in main.externalIpv6Hosts if h not in main.disconnectedExternalIpv6Hosts ], |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 129 | ipv6=True, acceptableFailed=7, |
You Wang | 54b1d67 | 2018-06-11 16:44:13 -0700 | [diff] [blame^] | 130 | stepMsg="Verify reachability from connected internal IPv6 hosts to external IPv6 hosts", |
| 131 | t3Simple=False ) |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 132 | # Verify disconnected hosts |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 133 | if disconnected: |
Jonghwan Hyun | 3759e47 | 2018-05-01 15:40:08 -0700 | [diff] [blame] | 134 | # Disconnected internal to connected external |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 135 | if main.disconnectedIpv4Hosts: |
Jonghwan Hyun | 3759e47 | 2018-05-01 15:40:08 -0700 | [diff] [blame] | 136 | lib.verifyPing( main, main.disconnectedIpv4Hosts, |
| 137 | [ h for h in main.externalIpv4Hosts if h not in main.disconnectedExternalIpv4Hosts ], |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 138 | expect=False, |
You Wang | 54b1d67 | 2018-06-11 16:44:13 -0700 | [diff] [blame^] | 139 | stepMsg="Verify unreachability of disconnected internal IPv4 hosts to connected external IPv4 hosts", |
| 140 | t3Simple=False ) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 141 | if main.disconnectedIpv6Hosts: |
Jonghwan Hyun | 3759e47 | 2018-05-01 15:40:08 -0700 | [diff] [blame] | 142 | lib.verifyPing( main, main.disconnectedIpv6Hosts, |
| 143 | [ h for h in main.externalIpv6Hosts if h not in main.disconnectedExternalIpv6Hosts ], |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 144 | ipv6=True, expect=False, |
You Wang | 54b1d67 | 2018-06-11 16:44:13 -0700 | [diff] [blame^] | 145 | stepMsg="Verify unreachability of disconnected internal IPv6 hosts to connected external IPv6 hosts", |
| 146 | t3Simple=False ) |
Jonghwan Hyun | 3759e47 | 2018-05-01 15:40:08 -0700 | [diff] [blame] | 147 | # Connected internal to disconnected external |
| 148 | if main.disconnectedExternalIpv4Hosts: |
| 149 | lib.verifyPing( main, |
| 150 | [ h for h in main.internalIpv4Hosts if h not in main.disconnectedIpv4Hosts ], |
| 151 | main.disconnectedExternalIpv4Hosts, |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 152 | expect=False, |
You Wang | 54b1d67 | 2018-06-11 16:44:13 -0700 | [diff] [blame^] | 153 | stepMsg="Verify unreachability of connected internal IPv4 hosts to disconnected external IPv4 hosts", |
| 154 | t3Simple=False ) |
Jonghwan Hyun | 3759e47 | 2018-05-01 15:40:08 -0700 | [diff] [blame] | 155 | if main.disconnectedExternalIpv6Hosts: |
| 156 | lib.verifyPing( main, |
| 157 | [ h for h in main.internalIpv6Hosts if h not in main.disconnectedIpv6Hosts ], |
| 158 | main.disconnectedExternalIpv6Hosts, |
You Wang | 8574776 | 2018-05-11 15:51:50 -0700 | [diff] [blame] | 159 | ipv6=True, expect=False, |
You Wang | 54b1d67 | 2018-06-11 16:44:13 -0700 | [diff] [blame^] | 160 | stepMsg="Verify unreachability of connected internal IPv6 hosts to disconnected external IPv6 hosts", |
| 161 | t3Simple=False ) |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 162 | |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 163 | def verifyPing( main, ipv4=True, ipv6=True, disconnected=False, internal=True, external=True ): |
You Wang | 5da39c8 | 2018-04-26 22:55:08 -0700 | [diff] [blame] | 164 | """ |
| 165 | Verify reachability and unreachability of connected/disconnected hosts |
| 166 | """ |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 167 | if internal: |
| 168 | verifyPingInternal( main, ipv4, ipv6, disconnected ) |
| 169 | if external: |
| 170 | verifyPingExternal( main, ipv4, ipv6, disconnected ) |
| 171 | |
Jon Hall | a604fd4 | 2018-05-04 14:27:27 -0700 | [diff] [blame] | 172 | def verifyLinkFailure( main, ipv4=True, ipv6=True, disconnected=False, |
| 173 | internal=True, external=True, countFlowsGroups=False ): |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 174 | """ |
| 175 | Kill and recover all links to spine101 and 102 sequencially and run verifications |
| 176 | """ |
| 177 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib |
| 178 | linksToRemove = [ ["spine103", "spine101"], |
| 179 | ["leaf2", "spine101"], |
| 180 | ["leaf3", "spine101"], |
| 181 | ["leaf4", "spine101"], |
| 182 | ["leaf5", "spine101"] ] |
| 183 | lib.killLinkBatch( main, linksToRemove, 30, 10 ) |
| 184 | verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups ) |
| 185 | lib.restoreLinkBatch( main, linksToRemove, 48, 10 ) |
| 186 | verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups ) |
| 187 | linksToRemove = [ ["spine104", "spine102"], |
| 188 | ["leaf2", "spine102"], |
| 189 | ["leaf3", "spine102"], |
| 190 | ["leaf4", "spine102"], |
| 191 | ["leaf5", "spine102"] ] |
| 192 | lib.killLinkBatch( main, linksToRemove, 30, 10 ) |
| 193 | verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups ) |
| 194 | lib.restoreLinkBatch( main, linksToRemove, 48, 10 ) |
| 195 | verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups ) |
| 196 | |
Jon Hall | a604fd4 | 2018-05-04 14:27:27 -0700 | [diff] [blame] | 197 | def verifySwitchFailure( main, ipv4=True, ipv6=True, disconnected=False, |
| 198 | internal=True, external=True, countFlowsGroups=False ): |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 199 | """ |
| 200 | Kill and recover spine101 and 102 sequencially and run verifications |
| 201 | """ |
| 202 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib |
| 203 | for switchToKill in [ "spine101", "spine102" ]: |
| 204 | lib.killSwitch( main, switchToKill, 9, 30 ) |
| 205 | verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups ) |
| 206 | lib.recoverSwitch( main, switchToKill, 10, 48 ) |
| 207 | verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups ) |
| 208 | |
Jon Hall | a604fd4 | 2018-05-04 14:27:27 -0700 | [diff] [blame] | 209 | def verifyOnosFailure( main, ipv4=True, ipv6=True, disconnected=False, |
| 210 | internal=True, external=True, countFlowsGroups=False ): |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 211 | """ |
| 212 | Kill and recover onos nodes sequencially and run verifications |
| 213 | """ |
| 214 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib |
| 215 | import json |
| 216 | import time |
| 217 | |
| 218 | numCtrls = len( main.Cluster.runningNodes ) |
| 219 | links = len( json.loads( main.Cluster.next().links() ) ) |
| 220 | switches = len( json.loads( main.Cluster.next().devices() ) ) |
Jon Hall | a604fd4 | 2018-05-04 14:27:27 -0700 | [diff] [blame] | 221 | mastershipSleep = float( main.params[ 'timers' ][ 'balanceMasterSleep' ] ) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 222 | for ctrl in xrange( numCtrls ): |
| 223 | # Kill node |
| 224 | lib.killOnos( main, [ ctrl ], switches, links, ( numCtrls - 1 ) ) |
| 225 | main.Cluster.active(0).CLI.balanceMasters() |
Jon Hall | a604fd4 | 2018-05-04 14:27:27 -0700 | [diff] [blame] | 226 | time.sleep( mastershipSleep ) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 227 | verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups ) |
| 228 | # Recover node |
| 229 | lib.recoverOnos( main, [ ctrl ], switches, links, numCtrls ) |
| 230 | main.Cluster.active(0).CLI.balanceMasters() |
Jon Hall | a604fd4 | 2018-05-04 14:27:27 -0700 | [diff] [blame] | 231 | time.sleep( mastershipSleep ) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 232 | verify( main, ipv4, ipv6, disconnected, internal, external, countFlowsGroups ) |
| 233 | |
| 234 | def verify( main, ipv4=True, ipv6=True, disconnected=True, internal=True, external=True, countFlowsGroups=False ): |
| 235 | """ |
| 236 | Verify host IP assignment, flow/group number and pings |
| 237 | """ |
| 238 | from tests.USECASE.SegmentRouting.dependencies.Testcaselib import Testcaselib as lib |
| 239 | # Verify host IP assignment |
| 240 | lib.verifyOnosHostIp( main ) |
| 241 | lib.verifyNetworkHostIp( main ) |
| 242 | # check flows / groups numbers |
| 243 | if countFlowsGroups: |
Jon Hall | a604fd4 | 2018-05-04 14:27:27 -0700 | [diff] [blame] | 244 | lib.checkFlowsGroupsFromFile( main ) |
You Wang | d66de19 | 2018-04-30 17:30:12 -0700 | [diff] [blame] | 245 | # ping hosts |
| 246 | verifyPing( main, ipv4, ipv6, disconnected, internal, external ) |