Retry failed app-Ids check

- Fix issue where we try to check app-ids when apps are still loading
- Add more debug logs

Change-Id: I5466ecd9bd7ca9d927f37bebc77e4b9a60aea628
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 0e5e376..db0b48c 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -3805,12 +3805,12 @@
                     result = main.FALSE
                 # get the entry in ids that has the same appID
                 current = filter( lambda item: item[ 'id' ] == appID, ids )
-                # main.log.debug( "Comparing " + str( app ) + " to " +
-                #                 str( current ) )
                 if not current:  # if ids doesn't have this id
                     result = main.FALSE
                     main.log.error( "'app-ids' does not have the ID for " +
                                     str( appName ) + " that apps does." )
+                    main.log.debug( "apps command returned: " + str( app ) +
+                                    "; app-ids has: " + str( ids ) )
                 elif len( current ) > 1:
                     # there is more than one app with this ID
                     result = main.FALSE
diff --git a/TestON/tests/HA/dependencies/HA.py b/TestON/tests/HA/dependencies/HA.py
index c75ccdf..c61cd4f 100644
--- a/TestON/tests/HA/dependencies/HA.py
+++ b/TestON/tests/HA/dependencies/HA.py
@@ -451,12 +451,21 @@
         Check App IDs on all nodes
         """
         # FIXME: Rename this to appIDCheck? or add a check for isntalled apps
-        appResults = main.Cluster.command( "appToIDCheck" )
-        appCheck = all( i == main.TRUE for i in appResults )
+        for i in range( 15 ):
+            # TODO modify retry or add a new version that accepts looking for
+            # a value in a return list instead of needing to match the entire
+            # return value to retry
+            appResults = main.Cluster.command( "appToIDCheck" )
+            appCheck = all( i == main.TRUE for i in appResults )
+            if appCheck:
+                break
+            else:
+                time.sleep( 5 )
+
         if not appCheck:
             ctrl = main.Cluster.active( 0 )
-            main.log.debug( "%s apps: %s" % ( ctrl.name, ctrl.apps() ) )
-            main.log.debug( "%s appIDs: %s" % ( ctrl.name, ctrl.appIDs() ) )
+            main.log.debug( "%s apps: %s" % ( ctrl.name, ctrl.pprint( ctrl.apps() ) ) )
+            main.log.debug( "%s appIDs: %s" % ( ctrl.name, ctrl.pprint( ctrl.appIDs() ) ) )
         return appCheck
 
     def workQueueStatsCheck( self, workQueueName, completed, inProgress, pending ):