Merge "Fix issues with disconnecting from cli or Errors in sendline"
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 345e0f9..1235353 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -399,25 +399,6 @@
             if level:
                 lvlStr = "--level=" + level
 
-            self.handle.sendline( "" )
-            i = self.handle.expect( [ "onos>", "\$", pexpect.TIMEOUT ] )
-            if i == 1:
-                main.log.error( self.name + ": onos cli session closed. ")
-                if self.onosIp:
-                    main.log.warn( "Trying to reconnect " + self.onosIp )
-                    reconnectResult = self.startOnosCli( self.onosIp )
-                    if reconnectResult:
-                        main.log.info( self.name + ": onos cli session reconnected." )
-                    else:
-                        main.log.error( self.name + ": reconnection failed." )
-                        main.cleanup()
-                        main.exit()
-                else:
-                    main.cleanup()
-                    main.exit()
-            if i == 2:
-                self.handle.sendline( "" )
-                self.handle.expect( "onos>" )
             self.handle.sendline( "log:log " + lvlStr + " " + cmdStr )
             self.handle.expect( "log:log" )
             self.handle.expect( "onos>" )
@@ -465,6 +446,27 @@
 
         """
         try:
+            # Try to reconnect if disconnected from cli
+            self.handle.sendline( "" )
+            i = self.handle.expect( [ "onos>", "\$", pexpect.TIMEOUT ] )
+            if i == 1:
+                main.log.error( self.name + ": onos cli session closed. ")
+                if self.onosIp:
+                    main.log.warn( "Trying to reconnect " + self.onosIp )
+                    reconnectResult = self.startOnosCli( self.onosIp )
+                    if reconnectResult:
+                        main.log.info( self.name + ": onos cli session reconnected." )
+                    else:
+                        main.log.error( self.name + ": reconnection failed." )
+                        main.cleanup()
+                        main.exit()
+                else:
+                    main.cleanup()
+                    main.exit()
+            if i == 2:
+                self.handle.sendline( "" )
+                self.handle.expect( "onos>" )
+
             if debug:
                 # NOTE: This adds and average of .4 seconds per call
                 logStr = "\"Sending CLI command: '" + cmdStr + "'\""
@@ -516,6 +518,7 @@
             return None
         except IndexError:
             main.log.exception( self.name + ": Object not as expected" )
+            main.log.debug( "response: {}".format( repr( response ) ) )
             return None
         except TypeError:
             main.log.exception( self.name + ": Object not as expected" )
@@ -558,6 +561,7 @@
             cmdStr = "add-node " + str( nodeId ) + " " +\
                 str( ONOSIp ) + " " + str( tcpPort )
             handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
             assert "Command not found:" not in handle, handle
             if re.search( "Error", handle ):
                 main.log.error( "Error in adding node" )
@@ -593,6 +597,7 @@
 
             cmdStr = "remove-node " + str( nodeId )
             handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
             assert "Command not found:" not in handle, handle
             if re.search( "Error", handle ):
                 main.log.error( "Error in removing node" )
@@ -628,6 +633,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             return output
         except AssertionError:
@@ -779,6 +785,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             return output
         except AssertionError:
@@ -3053,12 +3060,12 @@
             dpid = str( dpid )
             cmdStr = "onos:ports -e " + dpid + " | wc -l"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             if re.search( "No such device", output ):
                 main.log.error( "Error in getting ports" )
                 return ( output, "Error" )
-            else:
-                return output
+            return output
         except AssertionError:
             main.log.exception( "" )
             return None
@@ -3083,12 +3090,12 @@
             dpid = str( dpid )
             cmdStr = "onos:links " + dpid + " | grep ACTIVE | wc -l"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             if re.search( "No such device", output ):
                 main.log.error( "Error in getting ports " )
                 return ( output, "Error " )
-            else:
-                return output
+            return output
         except AssertionError:
             main.log.exception( "" )
             return None
@@ -3112,12 +3119,12 @@
         try:
             cmdStr = "onos:intents | grep id="
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             if re.search( "Error", output ):
                 main.log.error( "Error in getting ports" )
                 return ( output, "Error" )
-            else:
-                return output
+            return output
         except AssertionError:
             main.log.exception( "" )
             return None
@@ -3170,6 +3177,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             return output
         except AssertionError:
@@ -3199,6 +3207,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             return output
         except AssertionError:
@@ -3227,6 +3236,7 @@
         try:
             cmdStr = "onos:leaders -j"
             rawOutput = self.sendline( cmdStr )
+            assert rawOutput is not None, "Error in sendline"
             assert "Command not found:" not in rawOutput, rawOutput
             output = json.loads( rawOutput )
             results = []
@@ -3262,6 +3272,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             return output
         except AssertionError:
@@ -3300,6 +3311,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             return output
         except AssertionError:
@@ -3333,6 +3345,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             return output
@@ -3639,6 +3652,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             return output
@@ -3776,6 +3790,7 @@
             elif short:
                 baseStr += " -s"
             output = self.sendline( baseStr + cmdStr + componentStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             return output
@@ -3817,6 +3832,7 @@
             if value is not None:
                 cmdStr += " " + str( value )
             output = self.sendline( baseStr + cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             if value and check:
@@ -3868,6 +3884,7 @@
         try:
             cmdStr = "set-test-add " + str( setName ) + " " + str( values )
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             try:
                 # TODO: Maybe make this less hardcoded
@@ -3883,6 +3900,7 @@
                                "seconds before retrying." )
                 time.sleep( retryTime )  # Due to change in mastership
                 output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Error executing command" not in output
             positiveMatch = "\[(.*)\] was added to the set " + str( setName )
             negativeMatch = "\[(.*)\] was already in set " + str( setName )
@@ -3937,6 +3955,7 @@
                 cmdStr += str( setName ) + " " + str( values )
             output = self.sendline( cmdStr )
             try:
+                assert output is not None, "Error in sendline"
                 # TODO: Maybe make this less hardcoded
                 # ConsistentMap Exceptions
                 assert "org.onosproject.store.service" not in output
@@ -3950,6 +3969,7 @@
                                "seconds before retrying." )
                 time.sleep( retryTime )  # Due to change in mastership
                 output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             main.log.info( self.name + ": " + output )
@@ -4036,6 +4056,7 @@
             cmdStr += setName + " " + values
             output = self.sendline( cmdStr )
             try:
+                assert output is not None, "Error in sendline"
                 # TODO: Maybe make this less hardcoded
                 # ConsistentMap Exceptions
                 assert "org.onosproject.store.service" not in output
@@ -4049,6 +4070,7 @@
                                "seconds before retrying." )
                 time.sleep( retryTime )  # Due to change in mastership
                 output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             main.log.info( self.name + ": " + output )
@@ -4129,6 +4151,7 @@
             cmdStr += setName
             output = self.sendline( cmdStr )
             try:
+                assert output is not None, "Error in sendline"
                 # TODO: Maybe make this less hardcoded
                 # ConsistentMap Exceptions
                 assert "org.onosproject.store.service" not in output
@@ -4142,6 +4165,7 @@
                                "seconds before retrying." )
                 time.sleep( retryTime )  # Due to change in mastership
                 output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             main.log.info( self.name + ": " + output )
@@ -4195,6 +4219,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             main.log.info( self.name + ": " + output )
@@ -4235,6 +4260,7 @@
                 cmdStr += " " + str( delta )
             output = self.sendline( cmdStr )
             try:
+                assert output is not None, "Error in sendline"
                 # TODO: Maybe make this less hardcoded
                 # ConsistentMap Exceptions
                 assert "org.onosproject.store.service" not in output
@@ -4248,6 +4274,7 @@
                                "seconds before retrying." )
                 time.sleep( retryTime )  # Due to change in mastership
                 output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             main.log.info( self.name + ": " + output )
@@ -4297,6 +4324,7 @@
                 cmdStr += " " + str( delta )
             output = self.sendline( cmdStr )
             try:
+                assert output is not None, "Error in sendline"
                 # TODO: Maybe make this less hardcoded
                 # ConsistentMap Exceptions
                 assert "org.onosproject.store.service" not in output
@@ -4310,6 +4338,7 @@
                                "seconds before retrying." )
                 time.sleep( retryTime )  # Due to change in mastership
                 output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             assert "Error executing command" not in output, output
             main.log.info( self.name + ": " + output )
@@ -4351,6 +4380,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
             assert "Command not found:" not in handle, handle
             assert "Error:" not in handle, handle
             if not handle:
@@ -4390,6 +4420,7 @@
             cmdStr = "transactional-map-test-get "
             cmdStr += keyName
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             try:
                 # TODO: Maybe make this less hardcoded
@@ -4456,6 +4487,7 @@
             cmdStr = "transactional-map-test-put "
             cmdStr += numKeys + " " + value
             output = self.sendline( cmdStr )
+            assert output is not None, "Error in sendline"
             assert "Command not found:" not in output, output
             try:
                 # TODO: Maybe make this less hardcoded
@@ -4513,6 +4545,7 @@
             if jsonFormat:
                 cmdStr += " -j"
             handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
             assert "Command not found:" not in handle, handle
             return handle
         except AssertionError:
@@ -4540,6 +4573,7 @@
             if jsonFormat:
                 cmd += "-j "
             response = self.sendline( cmd + uri )
+            assert response is not None, "Error in sendline"
             assert "Command not found:" not in response, response
             return response
         except AssertionError:
@@ -4602,6 +4636,7 @@
                 else:
                     cmd += " {}:{}:{}".format( proto, item, port )
             response = self.sendline( cmd )
+            assert response is not None, "Error in sendline"
             assert "Command not found:" not in response, response
             if "Error" in response:
                 main.log.error( response )
@@ -4642,6 +4677,7 @@
             for d in device:
                 time.sleep( 1 )
                 response = self.sendline( "device-remove {}".format( d ) )
+                assert response is not None, "Error in sendline"
                 assert "Command not found:" not in response, response
                 if "Error" in response:
                     main.log.warn( "Error for device: {}\nResponse: {}".format( d, response ) )
@@ -4682,6 +4718,7 @@
             for h in host:
                 time.sleep( 1 )
                 response = self.sendline( "host-remove {}".format( h ) )
+                assert response is not None, "Error in sendline"
                 assert "Command not found:" not in response, response
                 if "Error" in response:
                     main.log.warn( "Error for host: {}\nResponse: {}".format( h, response ) )
@@ -4717,6 +4754,7 @@
         try:
             cmd =  "null-link null:{} null:{} {}".format( begin, end, state )
             response = self.sendline( cmd, showResponse=True )
+            assert response is not None, "Error in sendline"
             assert "Command not found:" not in response, response
             if "Error" in response or "Failure" in response:
                 main.log.error( response )