[ONOS-6584] Added retry function to getNetCFG functions in FUNCnetCfg and VPLS test cases.

Change-Id: I6b0b487562037e66f1fc229492c510024e9ee26b
diff --git a/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.py b/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.py
index d0eec88..51bbe38 100644
--- a/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.py
+++ b/TestON/tests/FUNC/FUNCnetCfg/FUNCnetCfg.py
@@ -46,7 +46,7 @@
             main.hostsData = {}
             main.nodes = []
             main.ONOSip = []
-            main.retrytimes = main.params[ 'RETRY' ]
+            main.retrytimes = int( main.params[ 'RETRY' ] )
             main.retrysleep = main.params[ 'RetrySleep' ]
             main.ONOSip = main.ONOSbench.getOnosIps()
 
@@ -437,10 +437,13 @@
         #Wait 5 secs after set up netCfg
         time.sleep( main.SetNetCfgSleep )
         if setS1Allow:
-            # Check what we set is what is in ONOS
-            getS1 = main.ONOSrest1.getNetCfg( subjectClass="devices",
-                                              subjectKey="of:0000000000000001",
-                                              configKey="basic" )
+            getS1 = utilities.retry( f=main.ONOSrest1.getNetCfg,
+                                     retValue=False,
+                                     kwargs={"subjectClass":"devices",
+                                             "subjectKey" : "of:0000000000000001",
+                                             "configKey" : "basic"},
+                                     attempts=main.retrytimes,
+                                     sleep=main.retrysleep )
             onosCfg = pprint( getS1 )
             sentCfg = pprint( s1Json )
             if onosCfg == sentCfg:
@@ -476,9 +479,13 @@
         time.sleep( main.SetNetCfgSleep )
         if setS3Disallow:
             # Check what we set is what is in ONOS
-            getS3 = main.ONOSrest1.getNetCfg( subjectClass="devices",
-                                              subjectKey="of:0000000000000003",
-                                              configKey="basic" )
+            getS3 = utilities.retry( f=main.ONOSrest1.getNetCfg,
+                                     retValue=False,
+                                     kwargs={"subjectClass": "devices",
+                                            "subjectKey": "of:0000000000000003",
+                                            "configKey": "basic"},
+                                     attempts=main.retrytimes,
+                                     sleep=main.retrysleep )
             onosCfg = pprint( getS3 )
             sentCfg = pprint( s3Json )
             if onosCfg == sentCfg:
@@ -581,9 +588,13 @@
         s2Result = False
         if setS2Allow:
             # Check what we set is what is in ONOS
-            getS2 = main.ONOSrest2.getNetCfg( subjectClass="devices",
-                                              subjectKey="of:0000000000000002",
-                                              configKey="basic" )
+            getS2 = utilities.retry( f=main.ONOSrest2.getNetCfg,
+                                     retValue=False,
+                                     kwargs={"subjectClass": "devices",
+                                            "subjectKey": "of:0000000000000002",
+                                            "configKey": "basic"},
+                                     attempts=main.retrytimes,
+                                     sleep=main.retrysleep )
             onosCfg = pprint( getS2 )
             sentCfg = pprint( s2Json )
             if onosCfg == sentCfg:
@@ -616,9 +627,14 @@
         s4Result = False
         if setS4Disallow:
             # Check what we set is what is in ONOS
-            getS4 = main.ONOSrest3.getNetCfg( subjectClass="devices",
-                                              subjectKey="of:0000000000000004",
-                                              configKey="basic" )
+            getS4 = utilities.retry( f=main.ONOSrest3.getNetCfg,
+                                     retValue=False,
+                                     kwargs={"subjectClass": "devices",
+                                            "subjectKey": "of:0000000000000004",
+                                            "configKey": "basic"},
+                                     attempts=main.retrytimes,
+                                     sleep=main.retrysleep )
+
             onosCfg = pprint( getS4 )
             sentCfg = pprint( s4Json )
             if onosCfg == sentCfg:
@@ -633,7 +649,6 @@
                                  actual=s4Result,
                                  onpass="Net Cfg added for device s4",
                                  onfail="Net Cfg for device s4 not correctly set" )
-
         main.netCfg.compareCfg( main, main.gossipTime )
 
     def CASE23( self, main ):
@@ -806,7 +821,13 @@
         main.netCfg.compareCfg( main, main.gossipTime )
 
         main.step( "Assert the net config for devices is empty" )
-        get = main.ONOSrest3.getNetCfg( subjectClass="devices" )
+
+        get = utilities.retry( f=main.ONOSrest3.getNetCfg,
+                               retValue = False,
+                               kwargs={"subjectClass":"devices"},
+                               sleep=main.retrysleep,
+                               attempts=main.retrytimes )
+
         utilities.assert_equals( expect='{}',
                                  actual=get,
                                  onpass="Successfully removed device config",
@@ -897,7 +918,11 @@
         pprint = main.nodes[ 0 ].pprint
         main.case( "Posting network configurations to the top level web resource" )
         main.step( "Get json object from Net Cfg" )
-        getinfo = main.ONOSrest1.getNetCfg()
+        getinfo = utilities.retry( f=main.ONOSrest1.getNetCfg,
+                                   retValue=False,
+                                   sleep=main.retrysleep,
+                                   attempts=main.retrytimes )
+
         main.log.debug( getinfo )
         main.step( "Posting json object to Net Cfg" )
         postinfo = main.ONOSrest1.setNetCfg( json.loads( getinfo ) )
@@ -926,8 +951,13 @@
                                                   subjectKey="of:0000000000000006", configKey="basic" )
         s6Result = False
         if setS6Disallow:
-            getS6 = main.ONOSrest1.getNetCfg( subjectClass="devices",
-                                              subjectKey="of:0000000000000006", configKey="basic" )
+            getS6 = utilities.retry( f=main.ONOSrest1.getNetCfg,
+                                     retValue=False,
+                                     kwargs={"subjectClass":"devices",
+                                            "subjectKey" : "of:0000000000000006",
+                                            "configKey" : "basic"},
+                                     sleep=main.retrysleep,
+                                     attempts=main.retrytimes )
             onosCfg = pprint( getS6 )
             sentCfg = pprint( s6Json )
             if onosCfg == sentCfg:
diff --git a/TestON/tests/FUNC/FUNCnetCfg/dependencies/netCfg.py b/TestON/tests/FUNC/FUNCnetCfg/dependencies/netCfg.py
index 39f32b1..62de12a 100644
--- a/TestON/tests/FUNC/FUNCnetCfg/dependencies/netCfg.py
+++ b/TestON/tests/FUNC/FUNCnetCfg/dependencies/netCfg.py
@@ -13,21 +13,34 @@
     if gossipTime:
         time.sleep( gossipTime * len( main.nodes ) )
     responses = []
-    failMsg = "Net Cfg is different on some nodes."
-    failed = False
+    result = utilities.retry( f=checkNodeResponses,
+                              retValue=False,
+                              kwargs={'main' : main,'responses' : responses},
+                              sleep = main.retrysleep,
+                              attempts = main.retrytimes )
+    utilities.assert_equals( expect=True,
+                             actual=result,
+                             onpass="Net Cfg is the same on all nodes",
+                             onfail="Check Net Cfg failed. Check above messages." )
+
+
+def checkNodeResponses ( main, responses ):
+    numberOfFailedNodes = 0  # Tracks the number of nodes that failed to get net configuration
     for node in main.nodes:
-        response = node.getNetCfg()
+        response = node.getNetCfg( )
         responses.append( node.pprint( response ) )
         if response == main.FALSE:
-            failed = True
+            numberOfFailedNodes += 1
+
     compare = [ i == responses[ 0 ] for i in responses ]
-    if failed:
-        failMsg += " Some nodes failed to GET netCfg."
-    utilities.assert_equals( expect=True,
-                             actual=all( compare ),
-                             onpass="Net Cfg is the same on all nodes",
-                             onfail=failMsg )
+    if numberOfFailedNodes == 0 and all( compare ):
+        return True
+
+    # Failed, providing feedback on cli
+    if numberOfFailedNodes > 0:
+        main.log.warn( numberOfFailedNodes + " node(s) failed to GET Net Config" )
     if not all( compare ):
-        main.log.debug( "Net Config results:" )
+        main.log.debug( "Net Cfg is different on some nodes. Net Config results:" )
         for i in responses:
             main.log.debug( i )
+    return False
diff --git a/TestON/tests/USECASE/VPLS/VPLSBasic/VPLSBasic.py b/TestON/tests/USECASE/VPLS/VPLSBasic/VPLSBasic.py
index d5d913e..76f7ccd 100644
--- a/TestON/tests/USECASE/VPLS/VPLSBasic/VPLSBasic.py
+++ b/TestON/tests/USECASE/VPLS/VPLSBasic/VPLSBasic.py
@@ -264,7 +264,10 @@
 
         main.step( "Check interface configurations" )
         result = False
-        getPorts = main.ONOSrest1.getNetCfg( subjectClass="ports" )
+        getPorts = utilities.retry( f=main.ONOSrest1.getNetCfg,
+                                    retValue=False,
+                                    kwargs={"subjectClass":"ports"},
+                                    sleep=SLEEP )
         onosCfg = pprint( getPorts )
         sentCfg = pprint( originalCfg.get( "ports" ) )
 
diff --git a/TestON/tests/USECASE/VPLS/dependencies/vpls.py b/TestON/tests/USECASE/VPLS/dependencies/vpls.py
index 729621d..02abba2 100644
--- a/TestON/tests/USECASE/VPLS/dependencies/vpls.py
+++ b/TestON/tests/USECASE/VPLS/dependencies/vpls.py
@@ -53,8 +53,10 @@
     clusterResult = True
     for node in main.RESTs:
         result = False
-        getVPLS = node.getNetCfg( subjectClass="apps",
-                                  subjectKey=app )
+        getVPLS = utilities.retry( f=node.getNetCfg,
+                                   retValue=False,
+                                   kwargs={"subjectClass":"apps", "subjectKey":app},
+                                   sleep=SLEEP )
         onosCfg = json.loads( getVPLS ).get( 'vpls' ).get( 'vplsList' )
         onosCfg = pprint( sanitizeConfig( onosCfg ) )
         sentCfg = pprint( sanitizeConfig( main.vplsConfig ) )