Merge "add fsfw port"
diff --git a/README.md b/README.md
index f84882a..6a8964b 100644
--- a/README.md
+++ b/README.md
@@ -11,14 +11,14 @@
 Setup
 -------------
 
-0. Pull the git repo from https://github.com/OPENNETWORKINGLAB/ONLabTest.git 
+0. Pull the git repo from https://github.com/OPENNETWORKINGLAB/OnosSystemTest.git 
 
-    $ git clone https://github.com/OPENNETWORKINGLAB/ONLabTest.git
+    $ git clone https://github.com/OPENNETWORKINGLAB/OnosSystemTest.git
 
 1. Make a symbolic link for TestON on the HOMEDIR 
    Execute the following from the home directory  
 
-    $ ln -s ONLabTest/TestON TestON
+    $ ln -s OnosSystemTest/TestON TestON
 
 2. Make sure python path is correct 
 
@@ -57,8 +57,12 @@
 
     1. Erlang R15B, R16B, R17 - if possible please use R17
 
+    $ sudo apt-get install erlang
+
     2. libpcap-dev package if eth interfaces will be used
 
+    $ sudo apt-get install libpcap-dev
+
     Building and Running:
 
     $ git clone https://github.com/shivarammysore/LINC-Switch.git linc-oe
diff --git a/TestON/core/utilities.py b/TestON/core/utilities.py
index cdf6c1a..ce502cb 100644
--- a/TestON/core/utilities.py
+++ b/TestON/core/utilities.py
@@ -35,6 +35,8 @@
 import email
 import os
 import email.mime.application
+import time
+import random
 
 class Utilities:
     '''
@@ -292,6 +294,53 @@
         else:
             return 0
 
+    def retry( self, f, retValue, args=(), kwargs={},
+               sleep=1, attempts=2, randomTime=False ):
+        """
+        Given a function and bad return values, retry will retry a function
+        until successful or give up after a certain number of attempts.
+
+        Arguments:
+        f        - a callable object
+        retValue - Return value(s) of f to retry on. This can be a list or an
+                   object.
+        args     - A tuple containing the arguments of f.
+        kwargs   - A dictionary containing the keyword arguments of f.
+        sleep    - Time in seconds to sleep between retries. If random is True,
+                   this is the max time to wait. Defaults to 1 second.
+        attempts - Max number of attempts before returning. If set to 1,
+                   f will only be called once. Defaults to 2 trys.
+        random   - Boolean indicating if the wait time is random between 0
+                   and sleep or exactly sleep seconds. Defaults to False.
+        """
+        # TODO: be able to pass in a conditional statement(s). For example:
+        #      retCondition = "< 7"
+        #      Then we do something like 'if eval( "ret " + retCondition ):break'
+        try:
+            assert attempts > 0, "attempts must be more than 1"
+            assert sleep >= 0, "sleep must be >= 0"
+            if not isinstance( retValue, list ):
+                retValue = [ retValue ]
+            for i in range( 0, attempts ):
+                ret = f( *args, **kwargs )
+                if ret not in retValue:
+                # NOTE that False in [ 0 ] == True
+                    break
+                if randomTime:
+                    sleeptime = random.randint( 0, sleep )
+                else:
+                    sleeptime = sleep
+                time.sleep( sleeptime )
+            return ret
+        except AssertionError:
+            main.log.exception( "Invalid arguements for retry: " )
+            main.cleanup()
+            main.exit()
+        except Exception:
+            main.log.exception( "Uncaught exception in retry: " )
+            main.cleanup()
+            main.exit()
+
 
 if __name__ != "__main__":
     import sys
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index cce13d3..eafc7ab 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -924,6 +924,16 @@
             if jsonFormat:
                 cmdStr += " -j"
             handle = self.sendline( cmdStr )
+            try:
+                # TODO: Maybe make this less hardcoded
+                # ConsistentMap Exceptions
+                assert "org.onosproject.store.service" not in handle
+                # Node not leader
+                assert "java.lang.IllegalStateException" not in handle
+            except AssertionError:
+                main.log.error( "Error in processing '" + cmdStr + "' " +
+                                "command: " + str( handle ) )
+                return None
             return handle
         except TypeError:
             main.log.exception( self.name + ": Object not as expected" )
diff --git a/TestON/tests/CHOtest/Dependencies/topoSpine.py b/TestON/tests/CHOtest/Dependencies/topoSpine.py
index cc681dd..5787d93 100755
--- a/TestON/tests/CHOtest/Dependencies/topoSpine.py
+++ b/TestON/tests/CHOtest/Dependencies/topoSpine.py
@@ -8,7 +8,6 @@
 from mininet.log import setLogLevel, info
 from mininet.link import TCLink, Intf
 from subprocess import call
-from mininet.log import setLogLevel
 from mininet.util import dumpNodeConnections
 from mininet.node import ( UserSwitch, OVSSwitch, IVSSwitch )
 
diff --git a/TestON/tests/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HAclusterRestart/HAclusterRestart.py
index ed8b2d2..2677b8d 100644
--- a/TestON/tests/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HAclusterRestart/HAclusterRestart.py
@@ -2181,9 +2181,11 @@
             ipResult = main.TRUE
             threads = []
             for i in range( main.numCtrls ):
-                t = main.Thread( target=main.CLIs[i].hosts,
+                t = main.Thread( target=utilities.retry,
                                  name="hosts-" + str( i ),
-                                 args=[ ] )
+                                 args=[ main.CLIs[i].hosts, [ None ] ],
+                                 kwargs= { 'sleep': 5, 'attempts': 5,
+                                           'randomTime': True } )
                 threads.append( t )
                 t.start()
 
@@ -2324,7 +2326,7 @@
                     elif i == 28:
                         deviceId = "2800".zfill(16)
                     mappings[ macId ] = deviceId
-                if hosts[ controller ] and "Error" not in hosts[ controller ]:
+                if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
                     if hosts[ controller ] == []:
                         main.log.warn( "There are no hosts discovered" )
                         noHosts = True
diff --git a/TestON/tests/HAkillNodes/HAkillNodes.py b/TestON/tests/HAkillNodes/HAkillNodes.py
index 411857e..4525713 100644
--- a/TestON/tests/HAkillNodes/HAkillNodes.py
+++ b/TestON/tests/HAkillNodes/HAkillNodes.py
@@ -2178,9 +2178,11 @@
             ipResult = main.TRUE
             threads = []
             for i in main.activeNodes:
-                t = main.Thread( target=main.CLIs[i].hosts,
+                t = main.Thread( target=utilities.retry,
                                  name="hosts-" + str( i ),
-                                 args=[ ] )
+                                 args=[ main.CLIs[i].hosts, [ None ] ],
+                                 kwargs= { 'sleep': 5, 'attempts': 5,
+                                           'randomTime': True } )
                 threads.append( t )
                 t.start()
 
@@ -2321,7 +2323,7 @@
                     elif i == 28:
                         deviceId = "2800".zfill(16)
                     mappings[ macId ] = deviceId
-                if hosts[ controller ] and "Error" not in hosts[ controller ]:
+                if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
                     if hosts[ controller ] == []:
                         main.log.warn( "There are no hosts discovered" )
                         zeroHosts = True
diff --git a/TestON/tests/HAsanity/HAsanity.py b/TestON/tests/HAsanity/HAsanity.py
index 1a5bbfe..340df87 100644
--- a/TestON/tests/HAsanity/HAsanity.py
+++ b/TestON/tests/HAsanity/HAsanity.py
@@ -2074,9 +2074,11 @@
             ipResult = main.TRUE
             threads = []
             for i in range( main.numCtrls ):
-                t = main.Thread( target=main.CLIs[i].hosts,
+                t = main.Thread( target=utilities.retry,
                                  name="hosts-" + str( i ),
-                                 args=[ ] )
+                                 args=[ main.CLIs[i].hosts, [ None ] ],
+                                 kwargs= { 'sleep': 5, 'attempts': 5,
+                                           'randomTime': True } )
                 threads.append( t )
                 t.start()
 
@@ -2217,7 +2219,7 @@
                     elif i == 28:
                         deviceId = "2800".zfill(16)
                     mappings[ macId ] = deviceId
-                if hosts[ controller ] and "Error" not in hosts[ controller ]:
+                if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
                     if hosts[ controller ] == []:
                         main.log.warn( "There are no hosts discovered" )
                         zeroHosts = True
diff --git a/TestON/tests/HAstopNodes/HAstopNodes.py b/TestON/tests/HAstopNodes/HAstopNodes.py
index cb5032b..2127d54 100644
--- a/TestON/tests/HAstopNodes/HAstopNodes.py
+++ b/TestON/tests/HAstopNodes/HAstopNodes.py
@@ -2167,9 +2167,11 @@
             ipResult = main.TRUE
             threads = []
             for i in main.activeNodes:
-                t = main.Thread( target=main.CLIs[i].hosts,
+                t = main.Thread( target=utilities.retry,
                                  name="hosts-" + str( i ),
-                                 args=[ ] )
+                                 args=[ main.CLIs[i].hosts, [ None ] ],
+                                 kwargs= { 'sleep': 5, 'attempts': 5,
+                                           'randomTime': True } )
                 threads.append( t )
                 t.start()
 
@@ -2310,7 +2312,7 @@
                     elif i == 28:
                         deviceId = "2800".zfill(16)
                     mappings[ macId ] = deviceId
-                if hosts[ controller ] and "Error" not in hosts[ controller ]:
+                if hosts[ controller ] is not None and "Error" not in hosts[ controller ]:
                     if hosts[ controller ] == []:
                         main.log.warn( "There are no hosts discovered" )
                         zeroHosts = True