[ONOS-7579] Investigate FUNCintent failures
- Inconsistently, some of the flows get stuck on Pending_remove after intents get removed. To prevent that, doing purge-intents separately
- Inconsistently, Encapsulation intent install shows different number of flows on each flow. To prevent test to exit for that, instead of looping by the number of flows, it will loop each flows. This way it will still set result as failure if flows don't look correct but keep the test on going.
Change-Id: Icbc502f0615af57f5472e26e83d2c0de7ef81507
diff --git a/TestON/tests/FUNC/FUNCintent/FUNCintent.py b/TestON/tests/FUNC/FUNCintent/FUNCintent.py
index daa17cb..86b9f53 100644
--- a/TestON/tests/FUNC/FUNCintent/FUNCintent.py
+++ b/TestON/tests/FUNC/FUNCintent/FUNCintent.py
@@ -564,7 +564,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -620,7 +620,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -649,7 +649,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -677,7 +677,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -706,7 +706,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -735,7 +735,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -856,7 +856,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -890,7 +890,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -925,7 +925,7 @@
protected=True,
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -959,7 +959,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1003,7 +1003,7 @@
expectedLink=18,
useTCP=True )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1068,7 +1068,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1165,7 +1165,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1199,7 +1199,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1233,7 +1233,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1353,7 +1353,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1394,7 +1394,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1434,8 +1434,7 @@
sw1="s5",
sw2="s2",
expectedLink=18 )
- else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1475,7 +1474,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1516,7 +1515,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1643,7 +1642,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1684,7 +1683,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1725,7 +1724,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1765,7 +1764,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1807,7 +1806,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1848,7 +1847,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -1965,7 +1964,7 @@
sw2="s2",
expectedLink=18 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -2033,7 +2032,7 @@
expectedLink1=16,
expectedLink2=14 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -2083,7 +2082,7 @@
expectedLink2=14,
partial=True )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -2130,7 +2129,7 @@
expectedLink1=16,
expectedLink2=14 )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
@@ -2180,7 +2179,7 @@
expectedLink2=14,
partial=True )
else:
- main.Cluster.active( 0 ).CLI.removeAllIntents( purge=True )
+ main.intents.removeAllExistIntents( main )
utilities.assert_equals( expect=main.TRUE,
actual=testResult,
diff --git a/TestON/tests/FUNC/FUNCintent/dependencies/FuncIntentFunction.py b/TestON/tests/FUNC/FUNCintent/dependencies/FuncIntentFunction.py
index afe4c4a..3f1514a 100644
--- a/TestON/tests/FUNC/FUNCintent/dependencies/FuncIntentFunction.py
+++ b/TestON/tests/FUNC/FUNCintent/dependencies/FuncIntentFunction.py
@@ -410,6 +410,17 @@
main.assertReturnString += 'Remove Intents Failed'
testResult = main.FALSE
+ # Check flows count in each node
+ if utilities.retry( f=checkFlowsState,
+ retValue=main.FALSE,
+ args=[ main ],
+ sleep=main.checkFlowCountSleep,
+ attempts=3 ):
+ main.assertReturnString += 'After removing intents Flow State Passed\n'
+ else:
+ main.assertReturnString += 'After removing intents Flow State Failed\n'
+ testResult = main.FALSE
+
return testResult
@@ -1477,6 +1488,17 @@
main.assertReturnString += 'Remove Intents Failed'
testResult = main.FALSE
+ # Check flows count in each node
+ if utilities.retry( f=checkFlowsState,
+ retValue=main.FALSE,
+ args=[ main ],
+ sleep=main.checkFlowCountSleep,
+ attempts=3 ):
+ main.assertReturnString += 'After removing intents Flow State Passed\n'
+ else:
+ main.assertReturnString += 'After removing intents Flow State Failed\n'
+ testResult = main.FALSE
+
return testResult
@@ -1896,6 +1918,17 @@
main.assertReturnString += 'Remove Intents Failed'
testResult = main.FALSE
+ # Check flows count in each node
+ if utilities.retry( f=checkFlowsState,
+ retValue=main.FALSE,
+ args=[ main ],
+ sleep=main.checkFlowCountSleep,
+ attempts=3 ):
+ main.assertReturnString += 'After removing intents Flow State Passed\n'
+ else:
+ main.assertReturnString += 'After removing intents Flow State Failed\n'
+ testResult = main.FALSE
+
return testResult
@@ -2217,6 +2250,26 @@
return connectionsFunctional
+def removeAllExistIntents( main ):
+ main.Cluster.active( 0 ).CLI.removeAllIntents()
+ main.log.debug( str( main.Cluster.active( 0 ).CLI.intents() ) )
+ main.log.info( "Sleeping {} seconds".format( main.removeIntentSleep ) )
+ time.sleep( main.removeIntentSleep )
+
+ main.Cluster.active( 0 ).CIL.purgeWithdrawnIntents()
+ main.log.debug( str( main.Cluster.active( 0 ).CLI.intents() ) )
+ main.log.info( "Sleeping {} seconds".format( main.removeIntentSleep ) )
+ time.sleep( main.removeIntentSleep )
+
+ # Check flows count in each node
+ if utilities.retry( f=checkFlowsState,
+ retValue=main.FALSE,
+ args=[ main ],
+ sleep=main.checkFlowCountSleep,
+ attempts=3 ):
+ main.assertReturnString += 'After removing intents Flow State Passed\n'
+ else:
+ main.assertReturnString += 'After removing intents Flow State Failed\n'
def removeAllIntents( main, intentsId ):
"""
@@ -2226,10 +2279,20 @@
removeIntentResult = main.TRUE
# Remove intents
for intent in intentsId:
- main.Cluster.active( 0 ).CLI.removeIntent( intentId=intent, purge=True )
-
- main.log.info( "Sleeping {} seconds".format( main.removeIntentSleep ) )
- time.sleep( main.removeIntentSleep )
+ main.Cluster.active( 0 ).CLI.removeIntent( intentId=intent )
+ main.log.debug( str( main.Cluster.active( 0 ).CLI.intents() ) )
+ main.log.info( "Sleeping {} seconds".format( 15 ) )
+ time.sleep( 15 )
+ try:
+ intents = json.loads( main.Cluster.active( 0 ).CLI.intents() )
+ for eachIntent in intents:
+ if eachIntent.get( 'id' ) == intent and eachIntent.get( 'state' ) == 'WITHDRAWN':
+ main.Cluster.active( 0 ).CLI.purgeWithdrawnIntents()
+ main.log.debug( str( main.Cluster.active( 0 ).CLI.intents() ) )
+ main.log.info( "Sleeping {} seconds".format( 15 ) )
+ time.sleep( 15 )
+ except:
+ main.log.warn( 'Intent does not exist' )
# If there is remianing intents then remove intents should fail
for ctrl in main.Cluster.active():
@@ -2425,9 +2488,9 @@
PushTag = tag + "_PUSH"
main.log.info( "Host Json info :" )
for EachHostJson in HostJson:
- for i in range( totalflows ):
- main.log.info( str( EachHostJson[ i ] ) )
- checkJson = EachHostJson[ i ][ "treatment" ][ "instructions" ][ 0 ]
+ for singleHostJson in EachHostJson:
+ main.log.info( str( singleHostJson ) )
+ checkJson = singleHostJson[ "treatment" ][ "instructions" ][ 0 ]
main.Cluster.active( 0 ).REST.pprint( checkJson )
if 'subtype' in checkJson:
if checkJson[ "subtype" ] == PopTag: