blob: f84b39cd37271ac19f7008780169fdf4434068a8 [file] [log] [blame]
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07001"""
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -07002Copyright 2016 Open Networking Foundation ( ONF )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -07003
4Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
5the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
6or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
7
8 TestON is free software: you can redistribute it and/or modify
9 it under the terms of the GNU General Public License as published by
10 the Free Software Foundation, either version 2 of the License, or
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070011 ( at your option ) any later version.
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -070012
13 TestON is distributed in the hope that it will be useful,
14 but WITHOUT ANY WARRANTY; without even the implied warranty of
15 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 GNU General Public License for more details.
17
18 You should have received a copy of the GNU General Public License
19 along with TestON. If not, see <http://www.gnu.org/licenses/>.
20"""
Piera2a7e1b2016-10-04 11:51:43 -070021# In this test we perform several failures and then test for connectivity
22# CASE1: 2x2 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070023# CASE2: 2x2 topo + 3 ONOS + | ONOS ( random instance ) failure + IP connectivity test | x failures
Piera2a7e1b2016-10-04 11:51:43 -070024# CASE3: 4x4 topo + 3 ONOS + | ONOS failure + IP connectivity test | x failures
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070025# CASE4: 4x4 topo + 3 ONOS + | ONOS ( random instance ) failure + IP connectivity test | x failures
Piera2a7e1b2016-10-04 11:51:43 -070026# CASE5: 2x2 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070027# CASE6: 2x2 topo + 3 ONOS + | ONOS ( random instance ) failure + Spine ( random switch ) failure + IP connectivity test | x failures
Piera2a7e1b2016-10-04 11:51:43 -070028# CASE7: 4x4 topo + 3 ONOS + | ONOS failure + Spine failure + IP connectivity test | x failures
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070029# CASE8: 4x4 topo + 3 ONOS + | ONOS ( random instance ) failure + Spine ( random switch ) failure + IP connectivity test | x failures
Piera2a7e1b2016-10-04 11:51:43 -070030
31
32class SRHighAvailability:
33
34 def __init__( self ):
35 self.default = ''
36
37 def CASE1( self, main ):
38 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070039 1 ) Sets up 3-nodes Onos-cluster
40 2 ) Start 2x2 Leaf-Spine topology
41 3 ) Pingall
42 4 ) Cause sequential ONOS failure
43 5 ) Pingall
44 6 ) Repeat 3 ), 4 ), 5 ) 'failures' times
Piera2a7e1b2016-10-04 11:51:43 -070045 """
46 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
47 Testcaselib as run
48 if not hasattr( main, 'apps' ):
49 run.initTest( main )
50
51 description = "High Availability tests - ONOS failures with 2x2 Leaf-spine "
52 main.case( description )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070053 run.config( main, '2x2' )
Piera2a7e1b2016-10-04 11:51:43 -070054 run.installOnos( main )
55 run.startMininet( main, 'cord_fabric.py' )
56 # pre-configured routing and bridging test
57 run.checkFlows( main, minFlowCount=116 )
58 run.pingAll( main )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070059 for i in range( 0, main.failures ):
Devin Lim142b5342017-07-20 15:22:39 -070060 toKill = i % main.Cluster.numCtrls
Piera2a7e1b2016-10-04 11:51:43 -070061 run.killOnos( main, [ toKill ], '4', '8', '2' )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070062 run.pingAll( main, 'CASE1_Failure%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -070063 run.recoverOnos( main, [ toKill ], '4', '8', '3' )
64 run.checkFlows( main, minFlowCount=116 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070065 run.pingAll( main, 'CASE1_Recovery%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -070066 run.cleanup( main )
67
68 def CASE2( self, main ):
69 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070070 1 ) Sets up 3-nodes Onos-cluster
71 2 ) Start 2x2 Leaf-Spine topology
72 3 ) Pingall
73 4 ) Cause random ONOS failure
74 5 ) Pingall
75 6 ) Repeat 3 ), 4 ), 5 ) 'failures' times
Piera2a7e1b2016-10-04 11:51:43 -070076 """
77 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
78 Testcaselib as run
79 import random
80 from random import randint
81 from datetime import datetime
82 if not hasattr( main, 'apps' ):
83 run.initTest( main )
84
85 description = "High Availability tests - ONOS random failures with 2x2 Leaf-spine "
86 main.case( description )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070087 run.config( main, '2x2' )
Piera2a7e1b2016-10-04 11:51:43 -070088 run.installOnos( main )
89 run.startMininet( main, 'cord_fabric.py' )
90 # pre-configured routing and bridging test
91 run.checkFlows( main, minFlowCount=116 )
92 run.pingAll( main )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070093 random.seed( datetime.now() )
94 for i in range( 0, main.failures ):
Devin Lim142b5342017-07-20 15:22:39 -070095 toKill = randint( 0, ( main.Cluster.numCtrls - 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -070096 run.killOnos( main, [ toKill ], '4', '8', '2' )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -070097 run.pingAll( main, 'CASE2_Failure%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -070098 run.recoverOnos( main, [ toKill ], '4', '8', '3' )
99 run.checkFlows( main, minFlowCount=116 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700100 run.pingAll( main, 'CASE2_Recovery%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700101 run.cleanup( main )
102
103 def CASE3( self, main ):
104 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700105 1 ) Sets up 3-nodes Onos-cluster
106 2 ) Start 4x4 Leaf-Spine topology
107 3 ) Pingall
108 4 ) Cause sequential ONOS failure
109 5 ) Pingall
110 6 ) Repeat 3 ), 4 ), 5 ) 'failures' times
Piera2a7e1b2016-10-04 11:51:43 -0700111 """
112 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
113 Testcaselib as run
114 if not hasattr( main, 'apps' ):
115 run.initTest( main )
116
117 description = "High Availability tests - ONOS failures with 4x4 Leaf-spine "
118 main.case( description )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700119 run.config( main, '4x4' )
Piera2a7e1b2016-10-04 11:51:43 -0700120 run.installOnos( main )
121 run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
122 # pre-configured routing and bridging test
123 run.checkFlows( main, minFlowCount=350 )
124 run.pingAll( main )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700125 for i in range( 0, main.failures ):
Devin Lim142b5342017-07-20 15:22:39 -0700126 toKill = i % main.Cluster.numCtrls
Piera2a7e1b2016-10-04 11:51:43 -0700127 run.killOnos( main, [ toKill ], '8', '32', '2' )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700128 run.pingAll( main, 'CASE3_Failure%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700129 run.recoverOnos( main, [ toKill ], '8', '32', '3' )
130 run.checkFlows( main, minFlowCount=350 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700131 run.pingAll( main, 'CASE3_Recovery%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700132 run.cleanup( main )
133
134 def CASE4( self, main ):
135 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700136 1 ) Sets up 3-nodes Onos-cluster
137 2 ) Start 4x4 Leaf-Spine topology
138 3 ) Pingall
139 4 ) Cause random ONOS failure
140 5 ) Pingall
141 6 ) Repeat 3 ), 4 ), 5 ) 'failures' times
Piera2a7e1b2016-10-04 11:51:43 -0700142 """
143 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
144 Testcaselib as run
145 import random
146 from random import randint
147 from datetime import datetime
148 if not hasattr( main, 'apps' ):
149 run.initTest( main )
150
151 description = "High Availability tests - ONOS random failures with 4x4 Leaf-spine "
152 main.case( description )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700153 run.config( main, '4x4' )
Piera2a7e1b2016-10-04 11:51:43 -0700154 run.installOnos( main )
155 run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
156 # pre-configured routing and bridging test
157 run.checkFlows( main, minFlowCount=350 )
158 run.pingAll( main )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700159 random.seed( datetime.now() )
160 for i in range( 0, main.failures ):
Devin Lim142b5342017-07-20 15:22:39 -0700161 toKill = randint( 0, ( main.Cluster.numCtrls - 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700162 run.killOnos( main, [ toKill ], '8', '32', '2' )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700163 run.pingAll( main, 'CASE4_Failure%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700164 run.recoverOnos( main, [ toKill ], '8', '32', '3' )
165 run.checkFlows( main, minFlowCount=350 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700166 run.pingAll( main, 'CASE4_Recovery%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700167 run.cleanup( main )
168
169 def CASE5( self, main ):
170 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700171 1 ) Sets up 3-nodes Onos-cluster
172 2 ) Start 2x2 Leaf-Spine topology
173 3 ) Pingall
174 4 ) Cause sequential ONOS failure
175 5 ) Pingall
176 6 ) Cause sequential Spine failure
177 7 ) Pingall
178 8 ) Repeat 3 ), 4 ), 5 ), 6 ), 7 ), 'failures' times
Piera2a7e1b2016-10-04 11:51:43 -0700179 """
180 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
181 Testcaselib as run
182 import time
183 if not hasattr( main, 'apps' ):
184 run.initTest( main )
185
186 description = "High Availability tests - ONOS failures and Switch failures with 2x2 Leaf-spine "
187 main.case( description )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700188 run.config( main, '2x2' )
Piera2a7e1b2016-10-04 11:51:43 -0700189 run.installOnos( main )
190 run.startMininet( main, 'cord_fabric.py' )
191 # pre-configured routing and bridging test
192 run.checkFlows( main, minFlowCount=116 )
193 run.pingAll( main )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700194 for i in range( 0, main.failures ):
Devin Lim142b5342017-07-20 15:22:39 -0700195 onosToKill = i % main.Cluster.numCtrls
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700196 switchToKill = i % len( main.spines )
Piera2a7e1b2016-10-04 11:51:43 -0700197 run.killOnos( main, [ onosToKill ], '4', '8', '2' )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700198 run.pingAll( main, 'CASE5_ONOS_Failure%d' % ( i + 1 ) )
199 run.killSwitch( main, main.spines[ switchToKill ][ 'name' ], switches='3', links='4' )
Piera2a7e1b2016-10-04 11:51:43 -0700200 time.sleep( main.switchSleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700201 run.pingAll( main, "CASE5_SWITCH_Failure%d" % ( i + 1 ) )
202 run.recoverSwitch( main, main.spines[ switchToKill ][ 'name' ], switches='4', links='8' )
Piera2a7e1b2016-10-04 11:51:43 -0700203 run.checkFlows( main, minFlowCount=116 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700204 run.pingAll( main, "CASE5_SWITCH_Recovery%d" % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700205 run.recoverOnos( main, [ onosToKill ], '4', '8', '3' )
206 run.checkFlows( main, minFlowCount=116 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700207 run.pingAll( main, 'CASE5_ONOS_Recovery%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700208 run.cleanup( main )
209
210 def CASE6( self, main ):
211 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700212 1 ) Sets up 3-nodes Onos-cluster
213 2 ) Start 2x2 Leaf-Spine topology
214 3 ) Pingall
215 4 ) Cause random ONOS failure
216 5 ) Pingall
217 6 ) Cause random Spine failure
218 7 ) Pingall
219 8 ) Repeat 3 ), 4 ), 5 ), 6 ), 7 ) 'failures' times
Piera2a7e1b2016-10-04 11:51:43 -0700220 """
221 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
222 Testcaselib as run
223 import time
224 import random
225 from random import randint
226 from datetime import datetime
227 if not hasattr( main, 'apps' ):
228 run.initTest( main )
229
230 description = "High Availability tests - ONOS random failures and Switch random failures with 2x2 Leaf-spine "
231 main.case( description )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700232 run.config( main, '2x2' )
Piera2a7e1b2016-10-04 11:51:43 -0700233 run.installOnos( main )
234 run.startMininet( main, 'cord_fabric.py' )
235 # pre-configured routing and bridging test
236 run.checkFlows( main, minFlowCount=116 )
237 run.pingAll( main )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700238 for i in range( 0, main.failures ):
Devin Lim142b5342017-07-20 15:22:39 -0700239 onosToKill = randint( 0, ( main.Cluster.numCtrls - 1 ) )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700240 switchToKill = randint( 0, 1 )
Piera2a7e1b2016-10-04 11:51:43 -0700241 run.killOnos( main, [ onosToKill ], '4', '8', '2' )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700242 run.pingAll( main, 'CASE6_ONOS_Failure%d' % ( i + 1 ) )
243 run.killSwitch( main, main.spines[ switchToKill ][ 'name' ], switches='3', links='4' )
Piera2a7e1b2016-10-04 11:51:43 -0700244 time.sleep( main.switchSleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700245 run.pingAll( main, "CASE6_SWITCH_Failure%d" % ( i + 1 ) )
246 run.recoverSwitch( main, main.spines[ switchToKill ][ 'name' ], switches='4', links='8' )
Piera2a7e1b2016-10-04 11:51:43 -0700247 run.checkFlows( main, minFlowCount=116 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700248 run.pingAll( main, "CASE6_SWITCH_Recovery%d" % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700249 run.recoverOnos( main, [ onosToKill ], '4', '8', '3' )
250 run.checkFlows( main, minFlowCount=116 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700251 run.pingAll( main, 'CASE6_ONOS_Recovery%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700252 run.cleanup( main )
253
254 def CASE7( self, main ):
255 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700256 1 ) Sets up 3-nodes Onos-cluster
257 2 ) Start 4x4 Leaf-Spine topology
258 3 ) Pingall
259 4 ) Cause sequential ONOS failure
260 5 ) Pingall
261 6 ) Cause sequential Spine failure
262 7 ) Pingall
263 8 ) Repeat 3 ), 4 ), 5 ), 6 ), 7 ), 'failures' times
Piera2a7e1b2016-10-04 11:51:43 -0700264 """
265 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
266 Testcaselib as run
267 import time
268 if not hasattr( main, 'apps' ):
269 run.initTest( main )
270
271 description = "High Availability tests - ONOS failures and Switch failures with 4x4 Leaf-spine "
272 main.case( description )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700273 run.config( main, '4x4' )
Piera2a7e1b2016-10-04 11:51:43 -0700274 run.installOnos( main )
275 run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
276 # pre-configured routing and bridging test
277 run.checkFlows( main, minFlowCount=350 )
278 run.pingAll( main )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700279 for i in range( 0, main.failures ):
Devin Lim142b5342017-07-20 15:22:39 -0700280 onosToKill = i % main.Cluster.numCtrls
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700281 switchToKill = i % len( main.spines )
Piera2a7e1b2016-10-04 11:51:43 -0700282 run.killOnos( main, [ onosToKill ], '8', '32', '2' )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700283 run.pingAll( main, 'CASE7_ONOS_Failure%d' % ( i + 1 ) )
284 run.killSwitch( main, main.spines[ switchToKill ][ 'name' ], switches='7', links='24' )
Piera2a7e1b2016-10-04 11:51:43 -0700285 time.sleep( main.switchSleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700286 run.pingAll( main, "CASE7_SWITCH_Failure%d" % ( i + 1 ) )
287 run.recoverSwitch( main, main.spines[ switchToKill ][ 'name' ], switches='8', links='32' )
Piera2a7e1b2016-10-04 11:51:43 -0700288 run.checkFlows( main, minFlowCount=350 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700289 run.pingAll( main, "CASE7_SWITCH_Recovery%d" % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700290 run.recoverOnos( main, [ onosToKill ], '8', '32', '3' )
291 run.checkFlows( main, minFlowCount=350 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700292 run.pingAll( main, 'CASE7_ONOS_Recovery%d' % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700293 run.cleanup( main )
294
295 def CASE8( self, main ):
296 """
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700297 1 ) Sets up 3-nodes Onos-cluster
298 2 ) Start 4x4 Leaf-Spine topology
299 3 ) Pingall
300 4 ) Cause random ONOS failure
301 5 ) Pingall
302 6 ) Cause random Spine failure
303 7 ) Pingall
304 8 ) Repeat 3 ), 4 ), 5 ), 6 ), 7 ), 'failures' times
Piera2a7e1b2016-10-04 11:51:43 -0700305 """
306 from tests.USECASE.SegmentRouting.dependencies.Testcaselib import \
307 Testcaselib as run
308 import time
309 import random
310 from random import randint
311 from datetime import datetime
312 if not hasattr( main, 'apps' ):
313 run.initTest( main )
314
315 description = "High Availability tests - ONOS random failures and Switch random failures with 4x4 Leaf-spine "
316 main.case( description )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700317 run.config( main, '4x4' )
Piera2a7e1b2016-10-04 11:51:43 -0700318 run.installOnos( main )
319 run.startMininet( main, 'cord_fabric.py', args="--leaf=4 --spine=4" )
320 # pre-configured routing and bridging test
321 run.checkFlows( main, minFlowCount=350 )
322 run.pingAll( main )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700323 for i in range( 0, main.failures ):
Devin Lim142b5342017-07-20 15:22:39 -0700324 onosToKill = randint( 0, ( main.Cluster.numCtrls - 1 ) )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700325 switchToKill = randint( 0, 3 )
Piera2a7e1b2016-10-04 11:51:43 -0700326 run.killOnos( main, [ onosToKill ], '8', '32', '2' )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700327 run.pingAll( main, 'CASE8_ONOS_Failure%d' % ( i + 1 ) )
328 run.killSwitch( main, main.spines[ switchToKill ][ 'name' ], switches='7', links='24' )
Piera2a7e1b2016-10-04 11:51:43 -0700329 time.sleep( main.switchSleep )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700330 run.pingAll( main, "CASE8_SWITCH_Failure%d" % ( i + 1 ) )
331 run.recoverSwitch( main, main.spines[ switchToKill ][ 'name' ], switches='8', links='32' )
Piera2a7e1b2016-10-04 11:51:43 -0700332 run.checkFlows( main, minFlowCount=350 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700333 run.pingAll( main, "CASE8_SWITCH_Recovery%d" % ( i + 1 ) )
Piera2a7e1b2016-10-04 11:51:43 -0700334 run.recoverOnos( main, [ onosToKill ], '8', '32', '3' )
335 run.checkFlows( main, minFlowCount=350 )
Jeremy Ronquillo23fb2162017-09-15 14:59:57 -0700336 run.pingAll( main, 'CASE8_ONOS_Recovery%d' % ( i + 1 ) )
Jeremy Ronquillob27ce4c2017-07-17 12:41:28 -0700337 run.cleanup( main )