Update tests for aether pods

- Update test for QA-POD
- SRStaging for testing connecting to Staging pod
- Add some functions for a kubernetes deployed cluster
- Connect to ONOS nodes with kubernetes
- Add option to connect to components through jump hosts
- Fixes for installing ONOS in custom locations
- Invoke python2 instead of python
- If using an ssh agent, also use that for pexpect ssh sessions,
  E.G. Jenkins initiated tests

Change-Id: I1fc345c8eab60a5b00c17e6ed677a63489a74a19
diff --git a/TestON/tests/dependencies/Cluster.py b/TestON/tests/dependencies/Cluster.py
index aa63714..2e79481 100644
--- a/TestON/tests/dependencies/Cluster.py
+++ b/TestON/tests/dependencies/Cluster.py
@@ -109,7 +109,7 @@
                 nodeList = self.runningNodes
 
         for ctrl in nodeList:
-            ips.append( ctrl.ipAddress )
+            ips.append( ctrl.ipAddress if ctrl.ipAddress is not 'localhost' else ctrl.address )
 
         return ips
 
@@ -310,18 +310,21 @@
         Returns:
             Returns main.TRUE if it successfully set and verify cell.
         """
+        result = main.TRUE
         setCellResult = self.command( "setCell",
                                       args=[ cellName ],
                                       specificDriver=1,
                                       getFrom="all" )
-        benchCellResult = main.ONOSbench.setCell( cellName )
-        verifyResult = self.command( "verifyCell",
-                                     specificDriver=1,
-                                     getFrom="all" )
-        result = main.TRUE
         for i in range( len( setCellResult ) ):
-            result = result and setCellResult[ i ] and verifyResult[ i ]
+            result = result and setCellResult[ i ]
+        benchCellResult = main.ONOSbench.setCell( cellName )
         result = result and benchCellResult
+        if not self.useDocker:
+            verifyResult = self.command( "verifyCell",
+                                         specificDriver=1,
+                                         getFrom="all" )
+            for i in range( len( verifyResult ) ):
+                result = result and verifyResult[ i ]
         return result
 
     def checkService( self ):
@@ -738,6 +741,7 @@
             Returns True if it successfully checked
         """
         results = True
+        self.command( "getAddress", specificDriver=2 )
         nodesOutput = self.command( "nodes", specificDriver=2 )
         ips = sorted( self.getIps( activeOnly=True ) )
         for i in nodesOutput:
diff --git a/TestON/tests/dependencies/ONOSSetup.py b/TestON/tests/dependencies/ONOSSetup.py
index 03275a1..042eff5 100644
--- a/TestON/tests/dependencies/ONOSSetup.py
+++ b/TestON/tests/dependencies/ONOSSetup.py
@@ -695,6 +695,7 @@
         elif main.persistentSetup:
             for ctrl in cluster.getRunningNodes():
                 ctrl.inDocker = True
+                ctrl.CLI.inDocker = True
 
         onosCliResult = main.TRUE
         if startOnosCli:
diff --git a/TestON/tests/dependencies/topology.py b/TestON/tests/dependencies/topology.py
index 7cdd9ab..26c3309 100644
--- a/TestON/tests/dependencies/topology.py
+++ b/TestON/tests/dependencies/topology.py
@@ -305,7 +305,8 @@
                     main.log.debug( "t3 command: {}".format( cmd ) )
                     main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, cmd, main.logdir,
                                                 "t3-CASE{}-{}-{}-route{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp, i ),
-                                                timeout=10 )
+                                                timeout=10,
+                                                cliPort=main.Cluster.active(0).CLI.karafPort )
         return main.FALSE if unexpectedPings else main.TRUE
 
     def sendScapyPackets( self, sender, receiver, pktFilter, pkt, sIface=None, dIface=None, expect=True, acceptableFailed=0, collectT3=True, t3Command="" ):
@@ -337,7 +338,8 @@
             main.log.debug( "Collecting t3 with source {} and destination {}".format( sender.name, receiver.name ) )
             main.log.debug( "t3 command: {}".format( t3Command ) )
             main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, t3Command, main.logdir,
-                                        "t3-CASE{}-{}-{}-".format( main.CurrentTestCaseNumber, sender.name, receiver.name ) )
+                                        "t3-CASE{}-{}-{}-".format( main.CurrentTestCaseNumber, sender.name, receiver.name ),
+                                        cliPort=main.Cluster.active(0).CLI.karafPort )
         return scapyResult
 
     def sendScapyPacketsHelper( self, sender, receiver, pktFilter, pkt, sIface=None, dIface=None, expect=True ):
@@ -418,7 +420,8 @@
                 main.log.debug( "t3 command: {}".format( cmd ) )
                 main.ONOSbench.dumpONOSCmd( main.Cluster.active( 0 ).ipAddress, cmd, main.logdir,
                                             "t3-CASE{}-{}-{}-route{}-".format( main.CurrentTestCaseNumber, srcIp, dstIp, i ),
-                                            timeout=10 )
+                                            timeout=10,
+                                            cliPort=main.Cluster.active(0).CLI.karafPort )
         return trafficResult
 
     def pingAndCaptureHelper( self, srcHost, dstIp, dstHost, dstIntf, ipv6=False, expect=True ):
diff --git a/TestON/tests/dependencies/utils.py b/TestON/tests/dependencies/utils.py
index 3cf849a..de61ae2 100644
--- a/TestON/tests/dependencies/utils.py
+++ b/TestON/tests/dependencies/utils.py
@@ -73,8 +73,12 @@
         stepResult = main.TRUE
         scpResult = main.TRUE
         copyResult = main.TRUE
+        isKube = False
         for ctrl in main.Cluster.runningNodes:
-            if ctrl.inDocker:
+            if ctrl.k8s:
+                isKube = True
+                continue
+            elif ctrl.inDocker:
                 scpResult = scpResult and ctrl.server.dockerCp( ctrl.name,
                                                                 "/opt/onos/log/karaf.log",
                                                                 "/tmp/karaf.log",
@@ -97,6 +101,28 @@
                 stepResult = main.TRUE and stepResult
             else:
                 stepResult = main.FALSE and stepResult
+        if isKube:
+            # TODO: Look into using Stern, kail, or just use `kubectl logs <pod>`
+            # We also need to save the pod name to switch name mapping
+            main.ONOSbench.kubectlPodNodes( dstPath=main.logdir + "/podMapping.txt",
+                                            kubeconfig=ctrl.k8s.kubeConfig,
+                                            namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
+            # TODO Get stratum write logs
+            # Save image for pods, based on "describe pods"
+            main.ONOSbench.kubectlDescribe( "pods",
+                                            main.logdir + "/describePods.txt",
+                                            kubeconfig=ctrl.k8s.kubeConfig,
+                                            namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
+            # Get the pod logs
+            pods = main.ONOSbench.kubectlGetPodNames( kubeconfig=ctrl.k8s.kubeConfig,
+                                                      namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
+
+            for pod in pods:
+                path = "%s/%s.log" % ( main.logdir, pod )
+                stratumPods = main.ONOSbench.kubectlLogs( pod,
+                                                          path,
+                                                          kubeconfig=ctrl.k8s.kubeConfig,
+                                                          namespace=main.params[ 'kubernetes' ][ 'namespace' ] )
         utilities.assert_equals( expect=main.TRUE,
                                  actual=stepResult,
                                  onpass="Successfully copied remote ONOS logs",