Merge pull request #103 from opennetworkinglab/devl/apps

Devl/apps
diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index baed719..a997933 100644
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -21,6 +21,7 @@
 import re
 import json
 import types
+import time
 sys.path.append( "../" )
 from drivers.common.clidriver import CLI
 
@@ -453,7 +454,6 @@
         Return:
             topology = current ONOS topology
         """
-        import json
         try:
             # either onos:topology or 'topology' will work in CLI
             cmdStr = "topology -j"
@@ -537,7 +537,7 @@
         """
         try:
             cmdStr = "device-remove "+str(deviceId)
-            handle = self.sendline( cmdStr )
+            self.sendline( cmdStr )
             return main.TRUE
         except TypeError:
             main.log.exception( self.name + ": Object not as expected" )
@@ -551,8 +551,6 @@
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanup()
             main.exit()
-        
-
 
     def devices( self, jsonFormat=True ):
         """
@@ -1207,11 +1205,11 @@
         """
         Note:
             This function assumes the format of all ingress devices
-            is same. That is, all ingress devices include port nos 
-            with a "/" or all ingress devices could specify device 
-            ids and port nos seperately.
+            is same. That is, all ingress devices include port numbers
+            with a "/" or all ingress devices could specify device
+            ids and port numbers seperately.
         Required:
-            * ingressDeviceList: List of device ids of ingress device 
+            * ingressDeviceList: List of device ids of ingress device
                 ( Atleast 2 ingress devices required in the list )
             * egressDevice: device id of egress device
         Optional:
@@ -1284,12 +1282,12 @@
                         cmd += " " + str( ingressDevice )
                     else:
                         main.log.error( "You must specify " +
-                                    "the ingress port" )
+                                        "the ingress port" )
                         # TODO: perhaps more meaningful return
                         return main.FALSE
             else:
                 if len( ingressDeviceList ) == len( portIngressList ):
-                    for ingressDevice,portIngress in zip( ingressDeviceList,portIngressList ):
+                    for ingressDevice, portIngress in zip( ingressDeviceList, portIngressList ):
                         cmd += " " + \
                             str( ingressDevice ) + "/" +\
                             str( portIngress ) + " "
@@ -1318,12 +1316,6 @@
                 main.log.info( "Multipoint-to-singlepoint intent installed" +
                                " failed " )
                 return None
-                #match = re.search('id=0x([\da-f]+),', handle)
-                #if match:
-                    #return match.group()[3:-1]
-                #else:
-                    #main.log.error( "Error, intent ID not found" )
-                    #return None
         except TypeError:
             main.log.exception( self.name + ": Object not as expected" )
             return None
@@ -1639,7 +1631,7 @@
             if handle:
                 return handle
             else:
-                # Return empty json 
+                # Return empty json
                 return '{}'
         except TypeError:
             main.log.exception( self.name + ": Object not as expected" )
@@ -2315,3 +2307,430 @@
             main.cleanup()
             main.exit()
 
+    def apps( self, jsonFormat=True ):
+        """
+        Returns the output of the apps command for ONOS. This command lists
+        information about installed ONOS applications
+        """
+        # Sample JSON object
+        # [{"name":"org.onosproject.openflow","id":0,"version":"1.2.0",
+        # "description":"ONOS OpenFlow protocol southbound providers",
+        # "origin":"ON.Lab","permissions":"[]","featuresRepo":"",
+        # "features":"[onos-openflow]","state":"ACTIVE"}]
+        try:
+            if jsonFormat:
+                cmdStr = "onos:apps -j"
+                output = self.sendline( cmdStr )
+                assert "Error executing command" not in output
+                ansiEscape = re.compile( r'\r\r\n\x1b[^m]*m' )
+                cleanedOutput = ansiEscape.sub( '', output )
+                return cleanedOutput
+            else:
+                cmdStr = "onos:apps"
+                output = self.sendline( cmdStr )
+                assert "Error executing command" not in output
+                return output
+        # FIXME: look at specific exceptions/Errors
+        except AssertionError:
+            main.log.error( "Error in processing onos:app command: " +
+                            str( output ) )
+            return None
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def appStatus( self, appName ):
+        """
+        Uses the onos:apps cli command to return the status of an application.
+        Returns:
+            "ACTIVE" - If app is installed and activated
+            "INSTALLED" - If app is installed and deactivated
+            "UNINSTALLED" - If app is not installed
+            None - on error
+        """
+        # FIXME also use app-ids to see if an uninstalled app is registered?
+        # FIXME "UNREGISTERED"?
+        try:
+            if not isinstance( appName, types.StringType ):
+                main.log.error( self.name + ".appStatus(): appName must be" +
+                                " a string" )
+                return None
+            output = self.apps( jsonFormat=True )
+            appsJson = json.loads( output )
+            state = None
+            for app in appsJson:
+                if appName == app.get('name'):
+                    state = app.get('state')
+                    break
+            if state == "ACTIVE" or state == "INSTALLED":
+                return state
+            elif state is None:
+                return "UNINSTALLED"
+            elif state:
+                main.log.error( "Unexpected state from 'onos:apps': " +
+                                str( state ) )
+                return state
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def app( self, appName, option ):
+        """
+        Interacts with the app command for ONOS. This command manages
+        application inventory.
+        """
+        try:
+            # Validate argument types
+            valid = True
+            if not isinstance( appName, types.StringType ):
+                main.log.error( self.name + ".app(): appName must be a " +
+                                "string" )
+                valid = False
+            if not isinstance( option, types.StringType ):
+                main.log.error( self.name + ".app(): option must be a string" )
+                valid = False
+            if not valid:
+                return main.FALSE
+            # Validate Option
+            option = option.lower()
+            # NOTE: Install may become a valid option
+            if option == "activate":
+                pass
+            elif option == "deactivate":
+                pass
+            elif option == "uninstall":
+                pass
+            else:
+                # Invalid option
+                main.log.error( "The ONOS app command argument only takes " +
+                                "the values: (activate|deactivate|uninstall)" +
+                                "; was given '" + option + "'")
+                return main.FALSE
+            cmdStr = "onos:app " + option + " " + appName
+            output = self.sendline( cmdStr )
+            if "Error executing command" in output:
+                main.log.error( "Error in processing onos:app command: " +
+                                str( output ) )
+                return main.FALSE
+            elif "No such application" in output:
+                main.log.error( "The application '" + appName +
+                                "' is not installed in ONOS" )
+                return main.FALSE
+            elif "Command not found:" in output:
+                main.log.error( "Error in processing onos:app command: " +
+                                str( output ) )
+                return main.FALSE
+            elif "Unsupported command:" in output:
+                main.log.error( "Incorrect command given to 'app': " +
+                                str( output ) )
+            # NOTE: we may need to add more checks here
+            # else: Command was successful
+            main.log.debug( "app response: " + repr( output ) )
+            return main.TRUE
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def activateApp( self, appName, check=True ):
+        """
+        Activate an app that is already installed in ONOS
+        appName is the hierarchical app name, not the feature name
+        If check is True, method will check the status of the app after the
+        command is issued
+        Returns main.TRUE if the command was successfully sent
+                main.FALSE if the cli responded with an error or given
+                    incorrect input
+        """
+        try:
+            if not isinstance( appName, types.StringType ):
+                main.log.error( self.name + ".activateApp(): appName must be" +
+                                " a string" )
+                return main.FALSE
+            status = self.appStatus( appName )
+            if status == "INSTALLED":
+                response = self.app( appName, "activate" )
+                if check and response == main.TRUE:
+                    for i in range(10):  # try 10 times then give up
+                        # TODO: Check with Thomas about this delay
+                        status = self.appStatus( appName )
+                        if status == "ACTIVE":
+                            return main.TRUE
+                        else:
+                            time.sleep( 1 )
+                    return main.FALSE
+                else:  # not 'check' or command didn't succeed
+                    return response
+            elif status == "ACTIVE":
+                return main.TRUE
+            elif status == "UNINSTALLED":
+                main.log.error( self.name + ": Tried to activate the " +
+                                "application '" + appName + "' which is not " +
+                                "installed." )
+            else:
+                main.log.error( "Unexpected return value from appStatus: " +
+                                str( status ) )
+                return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def deactivateApp( self, appName, check=True ):
+        """
+        Deactivate an app that is already activated in ONOS
+        appName is the hierarchical app name, not the feature name
+        If check is True, method will check the status of the app after the
+        command is issued
+        Returns main.TRUE if the command was successfully sent
+                main.FALSE if the cli responded with an error or given
+                    incorrect input
+        """
+        try:
+            if not isinstance( appName, types.StringType ):
+                main.log.error( self.name + ".deactivateApp(): appName must " +
+                                "be a string" )
+                return main.FALSE
+            status = self.appStatus( appName )
+            if status == "INSTALLED":
+                return main.TRUE
+            elif status == "ACTIVE":
+                response = self.app( appName, "deactivate" )
+                if check and response == main.TRUE:
+                    for i in range(10):  # try 10 times then give up
+                        status = self.appStatus( appName )
+                        if status == "INSTALLED":
+                            return main.TRUE
+                        else:
+                            time.sleep( 1 )
+                    return main.FALSE
+                else:  # not check or command didn't succeed
+                    return response
+            elif status == "UNINSTALLED":
+                main.log.warn( self.name + ": Tried to deactivate the " +
+                                "application '" + appName + "' which is not " +
+                                "installed." )
+                return main.TRUE
+            else:
+                main.log.error( "Unexpected return value from appStatus: " +
+                                str( status ) )
+                return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def uninstallApp( self, appName, check=True ):
+        """
+        Uninstall an app that is already installed in ONOS
+        appName is the hierarchical app name, not the feature name
+        If check is True, method will check the status of the app after the
+        command is issued
+        Returns main.TRUE if the command was successfully sent
+                main.FALSE if the cli responded with an error or given
+                    incorrect input
+        """
+        # TODO: check with Thomas about the state machine for apps
+        try:
+            if not isinstance( appName, types.StringType ):
+                main.log.error( self.name + ".uninstallApp(): appName must " +
+                                "be a string" )
+                return main.FALSE
+            status = self.appStatus( appName )
+            if status == "INSTALLED":
+                response = self.app( appName, "uninstall" )
+                if check and response == main.TRUE:
+                    for i in range(10):  # try 10 times then give up
+                        status = self.appStatus( appName )
+                        if status == "UNINSTALLED":
+                            return main.TRUE
+                        else:
+                            time.sleep( 1 )
+                    return main.FALSE
+                else:  # not check or command didn't succeed
+                    return response
+            elif status == "ACTIVE":
+                main.log.warn( self.name + ": Tried to uninstall the " +
+                                "application '" + appName + "' which is " +
+                                "currently active." )
+                response = self.app( appName, "uninstall" )
+                if check and response == main.TRUE:
+                    for i in range(10):  # try 10 times then give up
+                        status = self.appStatus( appName )
+                        if status == "UNINSTALLED":
+                            return main.TRUE
+                        else:
+                            time.sleep( 1 )
+                    return main.FALSE
+                else:  # not check or command didn't succeed
+                    return response
+            elif status == "UNINSTALLED":
+                return main.TRUE
+            else:
+                main.log.error( "Unexpected return value from appStatus: " +
+                                str( status ) )
+                return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def appIDs( self, jsonFormat=True ):
+        """
+        Show the mappings between app id and app names given by the 'app-ids'
+        cli command
+        """
+        try:
+            cmdStr = "app-ids"
+            if jsonFormat:
+                cmdStr += " -j"
+                output = self.sendline( cmdStr )
+                assert "Error executing command" not in output
+                ansiEscape = re.compile( r'\r\r\n\x1b[^m]*m' )
+                cleanedOutput = ansiEscape.sub( '', output )
+                return cleanedOutput
+            else:
+                output = self.sendline( cmdStr )
+                assert "Error executing command" not in output
+                return output
+        except AssertionError:
+            main.log.error( "Error in processing onos:app-ids command: " +
+                            str( output ) )
+            return None
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def appToIDCheck( self ):
+        """
+        This method will check that each application's ID listed in 'apps' is
+        the same as the ID listed in 'app-ids'. The check will also check that
+        there are no duplicate IDs issued. Note that an app ID should be
+        a globaly unique numerical identifier for app/app-like features. Once
+        an ID is registered, the ID is never freed up so that if an app is
+        reinstalled it will have the same ID.
+
+        Returns: main.TRUE  if the check passes and
+                 main.FALSE if the check fails or
+                 main.ERROR if there is some error in processing the test
+        """
+        try:
+            ids = json.loads( self.appIDs( jsonFormat=True ) )
+            apps = json.loads( self.apps( jsonFormat=True ) )
+            result = main.TRUE
+            for app in apps:
+                appID = app.get( 'id' )
+                if appID is None:
+                    main.log.error( "Error parsing app: " + str( app ) )
+                    result = main.FALSE
+                appName = app.get( 'name' )
+                if appName is None:
+                    main.log.error( "Error parsing app: " + str( app ) )
+                    result = main.FALSE
+                # get the entry in ids that has the same appID
+                current = filter(lambda item: item[ 'id' ] == appID, ids)
+                main.log.debug( "Comparing " + str( app ) + " to " +
+                                str( current ) )
+                if not current:  # if ids doesn't have this id
+                    result = main.FALSE
+                    main.log.error( "'app-ids' does not have the ID for " +
+                                    str( appName ) + " that apps does." )
+                elif len( current ) > 1:
+                    # there is more than one app with this ID
+                    result = main.FALSE
+                    # We will log this later in the method
+                elif not current[0][ 'name' ] == appName:
+                    currentName = current[0][ 'name' ]
+                    result = main.FALSE
+                    main.log.error( "'app-ids' has " + str( currentName ) +
+                                    " registered under id:" + str( appID ) +
+                                    " but 'apps' has " + str( appName ) )
+                else:
+                    pass  # id and name match!
+            # now make sure that app-ids has no duplicates
+            idsList = []
+            namesList = []
+            for item in ids:
+                idsList.append( item[ 'id' ] )
+                namesList.append( item[ 'name' ] )
+            if len( idsList ) != len( set( idsList ) ) or\
+               len( namesList ) != len( set( namesList ) ):
+                    main.log.error( "'app-ids' has some duplicate entries: \n"
+                                    + json.dumps( ids,
+                                                  sort_keys=True,
+                                                  indent=4,
+                                                  separators=( ',', ': ' ) ) )
+                    result = main.FALSE
+            return result
+        except ( ValueError, TypeError ):
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
diff --git a/TestON/drivers/common/cli/onosdriver.py b/TestON/drivers/common/cli/onosdriver.py
index d94c0e4..f96a60e 100644
--- a/TestON/drivers/common/cli/onosdriver.py
+++ b/TestON/drivers/common/cli/onosdriver.py
@@ -532,7 +532,7 @@
             main.exit()
 
     def createCellFile( self, benchIp, fileName, mnIpAddrs,
-                        extraFeatureString, *onosIpAddrs ):
+                        appString, *onosIpAddrs ):
         """
         Creates a cell file based on arguments
         Required:
@@ -556,11 +556,11 @@
         # Create the cell file in the directory for writing ( w+ )
         cellFile = open( tempDirectory + fileName, 'w+' )
 
-        # Feature string is hardcoded environment variables
+        # App string is hardcoded environment variables
         # That you may wish to use by default on startup.
-        # Note that you  may not want certain features listed
+        # Note that you  may not want certain apps listed
         # on here.
-        coreFeatureString = "export ONOS_FEATURES=" + extraFeatureString
+        appString = "export ONOS_APPS=" + appString
         mnString = "export OCN="
         onosString = "export OC"
         tempCount = 1
@@ -589,7 +589,7 @@
                 tempCount = tempCount + 1
 
             cellFile.write( mnString + "\"" + mnIpAddrs + "\"" + "\n" )
-            cellFile.write( coreFeatureString + "\n" )
+            cellFile.write( appString + "\n" )
             cellFile.close()
 
             # We use os.system to send the command to TestON cluster
@@ -1729,37 +1729,54 @@
         os.system( "scp " + tempFile + " admin@" + benchIp + ":" + linkGraphPath)        
         main.log.info("linkGraph.cfg creation complete")
 
-    def createNullDevProviderFile( self, benchIp, ONOSIpList, deviceCount, numPorts=10):
+    def configNullDev( self, ONOSIpList, deviceCount, numPorts=10):
         
         '''
-            benchIp = Ip address of the test bench 
             ONOSIpList = list of Ip addresses of nodes switches will be devided amongst 
-            deviceCount = number of switches to distribute 
-            numPorts = number of ports per device, when not specified in file it defaults to 10, optional arg
+            deviceCount = number of switches to distribute, or list of values to use as custom distribution  
+            numPorts = number of ports per device. Defaults to 10 both in this function and in ONOS. Optional arg
         '''
 
-        main.log.step("Creating null device provider configuration file." )
-        nullDevicePath = self.home + "/tools/package/etc/org.onosproject.provider.nil.device.impl.NullDeviceProvider.cfg"
-        tempFile = "/tmp/org.onosproject.provider.nil.device.impl.NullDeviceProvider.cfg"
-        configFile = open(tempFile, 'w+')
+        main.log.step("Configuring Null Device Provider" )
         clusterCount = len(ONOSIpList)
 
-        if type(deviceCount) is int or type(deviceCount) is str:
-            main.log.info("Creating device distribution")
-            deviceCount = int(deviceCount)
-            switchList = [0]*(clusterCount+1)
-            baselineSwitchCount = deviceCount/clusterCount
+        try: 
+            
+            if type(deviceCount) is int or type(deviceCount) is str:
+                main.log.step("Creating device distribution")
+                deviceCount = int(deviceCount)
+                switchList = [0]*(clusterCount+1)
+                baselineSwitchCount = deviceCount/clusterCount
 
-            for node in range(1, clusterCount + 1):
-                switchList[node] = baselineSwitchCount
+                for node in range(1, clusterCount + 1):
+                    switchList[node] = baselineSwitchCount
 
-            for node in range(1, (deviceCount%clusterCount)+1):
-                switchList[node] += 1
+                for node in range(1, (deviceCount%clusterCount)+1):
+                    switchList[node] += 1
+                
+            if type(deviceCount) is list: 
+                main.log.info("Using provided device distribution") 
+                
+                if len(deviceCount) == clusterCount: 
+                    switchList = ['0']
+                    switchList.extend(deviceCount)
+                
+                if len(deviceCount) == (clusterCount + 1): 
+                    if deviceCount[0] == '0' or deviceCount[0] == 0: 
+                        switchList = deviceCount
 
-        if type(deviceCount) is list: 
-            main.log.info("Using provided device distribution") 
-            switchList = ['0']
-            switchList.extend(deviceCount)
+                assert len(switchList) == (clusterCount + 1)
+        
+        except AssertionError:
+            main.log.error( "Bad device/Ip list match") 
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
 
         ONOSIp = [0]
         ONOSIp.extend(ONOSIpList)
@@ -1769,57 +1786,69 @@
             devicesString += (ONOSIp[node] + ":" + str(switchList[node] ))
             if node < clusterCount:
                 devicesString += (",")
+        
+        try: 
+            self.handle.sendline("onos $OC1 cfg set org.onosproject.provider.nil.device.impl.NullDeviceProvider devConfigs " + devicesString )
+            self.handle.expect(":~")
+            self.handle.sendline("onos $OC1 cfg set org.onosproject.provider.nil.device.impl.NullDeviceProvider numPorts " + str(numPorts) )
+            self.handle.expect(":~")
 
-        configFile.write(devicesString + "\n")
-        if numPorts == 10:
-            configFile.write("#numPorts = 10")
-        else: 
-            configFile.write("numPorts = " + str(numPorts))
+            for i in range(10):
+                self.handle.sendline("onos $OC1 cfg get org.onosproject.provider.nil.device.impl.NullDeviceProvider")
+                self.handle.expect(":~")
+                verification = self.handle.before
+                if (" value=" + str(numPorts)) in verification and (" value=" + devicesString) in verification:
+                    break
+                else:
+                    time.sleep(1)
 
-        configFile.close()        
-        os.system( "scp " + tempFile + " admin@" + benchIp + ":" + nullDevicePath)
+            assert ("value=" + str(numPorts)) in verification and (" value=" + fileName) in verification
+        
+        except AssertionError:
+            main.log.error("Incorrect Config settings: " + verification)
 
-    def createNullLinkProviderFile( self, benchIp, neighborIpList=0,fileName="", eventRate=0, onNode=False): 
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanup()
+            main.exit()
+
+    def configNullLink( self,fileName="/opt/onos/apache-karaf-3.0.2/etc/linkGraph.cfg", eventRate=0): 
         '''
-                neighbor list is an optional list of neighbors to be written directly to the file
-                onNode - bool, if true, alternate file path will be used to scp, inteneded
-                        for use on cell
+                fileName default is currently the same as the default on ONOS, specify alternate file if 
+                you want to use a different topology file than linkGraph.cfg
         '''
 
-        main.log.step("Creating Null Link Provider config file")
-        nullLinkPath = self.home + "/tools/package/etc/org.onosproject.provider.nil.link.impl.NullLinkProvider.cfg"
-        if onNode == True: 
-            nullLinkPath = "/opt/onos/apache-karaf-3.0.2/etc/org.onosproject.provider.nil.link.impl.NullLinkProvider.cfg"
-        tempFile = "/tmp/org.onosproject.provider.nil.link.impl.NullLinkProvider.cfg"
-        configFile = open(tempFile, 'w+')
-    
-        eventRate = int(eventRate)
-
-        if eventRate == 0: 
-            configFile.write("#eventRate = \n")
-        else: 
-            configFile.write("eventRate = " + str(eventRate) + "\n") 
         
-        if fileName == "":
-            configFile.write("#cfgFile = /tmp/foo.cfg        #If enabled, points to the full path to the topology file.\n") 
-        else: 
-            configFile.write("cfgFile =" + str(fileName) + "\n")
-
-        if neighborIpList != 0:
-            configFile.write("neighbors = ")
-            for n in range (0, len(neighborIpList)):
-                configFile.write(neighborIpList[n])
-                if n < (len(neighborIpList) - 1):
-                    configFile.write(",")            
-        else: 
-            configFile.write("#neighbors = ") 
+        try: 
+            self.handle.sendline("onos $OC1 cfg set org.onosproject.provider.nil.link.impl.NullLinkProvider eventRate " + str(eventRate)) 
+            self.handle.expect(":~")            
+            self.handle.sendline("onos $OC1 cfg set org.onosproject.provider.nil.link.impl.NullLinkProvider cfgFile " + fileName )
+            self.handle.expect(":~")
+               
+            for i in range(10): 
+                self.handle.sendline("onos $OC1 cfg get org.onosproject.provider.nil.link.impl.NullLinkProvider") 
+                self.handle.expect(":~")
+                verification = self.handle.before
+                if (" value=" + str(eventRate)) in verification and (" value=" + fileName) in verification: 
+                    break
+                else: 
+                    time.sleep(1)
+            
+            assert ("value=" + str(eventRate)) in verification and (" value=" + fileName) in verification
+            
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanup()
+            main.exit()
         
-        configFile.close()
-        if onNode == False:
-            os.system( "scp " + tempFile + " admin@" + benchIp + ":" + nullLinkPath)
-        if onNode == True:
-            os.system( "scp " + tempFile + " sdn@" + benchIp + ":" + nullLinkPath)
-        
+        except AssertionError:
+            main.log.info("Settings did not post to ONOS")
+            main.log.error(varification)            
 
-
+        except:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.log.error(varification)
+            main.cleanup()
+            main.exit()
 
diff --git a/TestON/tests/PingallExample/PingallExample.params b/TestON/tests/PingallExample/PingallExample.params
index 12bebcd..1e4cfc1 100644
--- a/TestON/tests/PingallExample/PingallExample.params
+++ b/TestON/tests/PingallExample/PingallExample.params
@@ -1,12 +1,12 @@
 <PARAMS>
     <testcases>1,2,3</testcases>
     <ENV>
-        <cellName>kelvin</cellName>
+        <cellName>SingleHA</cellName>
     </ENV>
     <Git>xe</Git>
 
     <CTRL>
-        <ip1>10.128.10.21</ip1>
+        <ip1>10.128.30.11</ip1>
         <port1>6633</port1>
     </CTRL>
 </PARAMS>
diff --git a/TestON/tests/PingallExample/PingallExample.py b/TestON/tests/PingallExample/PingallExample.py
index dab380f..c03b0f1 100644
--- a/TestON/tests/PingallExample/PingallExample.py
+++ b/TestON/tests/PingallExample/PingallExample.py
@@ -94,7 +94,7 @@
         if case1Result == main.FALSE:
             main.cleanup()
             main.exit()
-        
+
         # Starting the mininet using the old way
         main.step( "Starting Mininet ..." )
         netIsUp = main.Mininet1.startNet()
@@ -143,7 +143,7 @@
 
     def CASE3( self, main ):
         """
-           Assign intents
+           Install forwarding app, Pingall and unistall the app
         """
         import time
 
@@ -151,10 +151,11 @@
         main.case( "Run Pingall" )
 
         # install onos-app-fwd
-        main.log.info( "Install reactive forwarding app" )
-        main.ONOScli1.featureInstall( "onos-app-fwd" )
+        main.step( "Activate reactive forwarding app" )
+        main.ONOScli1.activateApp( "org.onosproject.fwd" )
 
         # REACTIVE FWD test
+        main.step( "Run the pingall command in Mininet" )
         pingResult = main.FALSE
         time1 = time.time()
         pingResult = main.Mininet1.pingall()
@@ -162,8 +163,8 @@
         main.log.info( "Time for pingall: %2f seconds" % ( time2 - time1 ) )
 
         # uninstall onos-app-fwd
-        main.log.info( "Uninstall reactive forwarding app" )
-        main.ONOScli1.featureUninstall( "onos-app-fwd" )
+        main.step( "Deactivate reactive forwarding app" )
+        main.ONOScli1.deactivateApp( "org.onosproject.fwd" )
 
         utilities.assert_equals( expect=main.TRUE, actual=pingResult,
                                  onpass="All hosts are reachable",
diff --git a/TestON/tests/PingallExample/PingallExample.topo b/TestON/tests/PingallExample/PingallExample.topo
index dba7a5d..3eda540 100644
--- a/TestON/tests/PingallExample/PingallExample.topo
+++ b/TestON/tests/PingallExample/PingallExample.topo
@@ -2,7 +2,7 @@
     <COMPONENT>
 
         <ONOSbench>
-            <host>10.128.10.20</host>
+            <host>10.128.30.10</host>
             <user>admin</user>
             <password></password>
             <type>OnosDriver</type>
@@ -11,7 +11,7 @@
         </ONOSbench>
 
         <ONOScli1>
-            <host>10.128.10.20</host>
+            <host>10.128.30.10</host>
             <user>admin</user>
             <password></password>
             <type>OnosCliDriver</type>
@@ -20,7 +20,7 @@
         </ONOScli1>
 
         <ONOS1>
-            <host>10.128.10.21</host>
+            <host>10.128.30.11</host>
             <user>admin</user>
             <password></password>
             <type>OnosDriver</type>
@@ -29,7 +29,7 @@
         </ONOS1>
 
         <Mininet1>
-            <host>10.128.10.20</host>
+            <host>10.128.30.9</host>
             <user>admin</user>
             <password></password>
             <type>MininetCliDriver</type>
diff --git a/TestON/tests/ScaleOutTemplate/.ScaleOutTemplate.py.swo b/TestON/tests/ScaleOutTemplate/.ScaleOutTemplate.py.swo
deleted file mode 100644
index a1a28ae..0000000
--- a/TestON/tests/ScaleOutTemplate/.ScaleOutTemplate.py.swo
+++ /dev/null
Binary files differ
diff --git a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.params b/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.params
index 641d16a..d4fa651 100644
--- a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.params
+++ b/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.params
@@ -1,13 +1,13 @@
 <PARAMS>
 
-    <testcases>1,2</testcases>
+    <testcases>1,2,1,2,1,2,1,2</testcases>
 
-    <SCALE>2</SCALE>
+    <SCALE>1,3,5,7</SCALE>
     <availableNodes>7</availableNodes>
  
     <ENV>
         <cellName>defaultCell</cellName>
-        <cellFeatures></cellFeatures>
+        <cellApps></cellApps>
     </ENV>
 
     <TEST>
@@ -46,7 +46,7 @@
     </CTRL>
 
     <MN>
-        <ip1>10.128.5.59</ip1>
+        <ip1>10.128.5.55</ip1>
     </MN>
 
     <BENCH>
diff --git a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py b/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py
index fdcd680..c290155 100644
--- a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py
+++ b/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.py
@@ -16,63 +16,46 @@
     def CASE1( self, main ):            #This is the initialization case
                                         #this case will clean up all nodes 
         import time                     #but only node 1 is started in this case
-        
-        global clusterCount             #number of nodes running
-        global ONOSIp                   #list of ONOS IP addresses 
-        clusterCount = 1
-        ONOSIp = [ 0 ]
-
-
+        global init       
+        try: 
+            if type(init) is not bool: 
+                init = False  
+        except NameError: 
+            init = False 
+       
         #Load values from params file
         checkoutBranch = main.params[ 'GIT' ][ 'checkout' ]
         gitPull = main.params[ 'GIT' ][ 'autopull' ]
         cellName = main.params[ 'ENV' ][ 'cellName' ]
-        Features= main.params[ 'ENV' ][ 'cellFeatures' ]
+        Apps = main.params[ 'ENV' ][ 'cellApps' ]
         BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
         BENCHUser = main.params[ 'BENCH' ][ 'user' ]
         MN1Ip = main.params[ 'MN' ][ 'ip1' ]
         maxNodes = int(main.params[ 'availableNodes' ])
-        Features = main.params[ 'ENV' ][ 'cellFeatures' ]
         skipMvn = main.params[ 'TEST' ][ 'skipCleanInstall' ]
+        cellName = main.params[ 'ENV' ][ 'cellName' ]        
 
-        #Populate ONOSIp with ips from params 
-        for i in range(1, maxNodes + 1): 
-            ipString = 'ip' + str(i) 
-            ONOSIp.append(main.params[ 'CTRL' ][ ipString ])   
+        # -- INIT SECTION, ONLY RUNS ONCE -- # 
+        if init == False: 
+            init = True
+            global clusterCount             #number of nodes running
+            global ONOSIp                   #list of ONOS IP addresses
+            global scale 
+            
+            clusterCount = 0
+            ONOSIp = [ 0 ]
+            scale = (main.params[ 'SCALE' ]).split(",")            
+            clusterCount = int(scale[0])
 
-        #############################
-        tempIp = [ ONOSIp[1],ONOSIp[2],ONOSIp[3],ONOSIp[4],ONOSIp[5]]
-        main.ONOSbench.createLinkGraphFile(BENCHIp, tempIp, str(7)) 
+            #Populate ONOSIp with ips from params 
+            for i in range(1, maxNodes + 1): 
+                ipString = 'ip' + str(i) 
+                ONOSIp.append(main.params[ 'CTRL' ][ ipString ])   
+            
+            #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
+            if skipMvn != "yes":
+                mvnResult = main.ONOSbench.cleanInstall()
 
-        main.log.info("marker")
-        #############################
-
-
-        #kill off all onos processes 
-        main.log.step("Safety check, killing all ONOS processes")
-        main.log.step("before initiating enviornment setup")
-        for node in range(1, maxNodes + 1):
-            main.ONOSbench.onosDie(ONOSIp[node])
-
-
-        #construct the cell file
-        main.log.info("Creating cell file")
-        exec "a = main.ONOSbench.createCellFile"
-        cellIp = []
-        for node in range (1, maxNodes + 1):
-            cellIp.append(ONOSIp[node])
-        a(BENCHIp,cellName,MN1Ip,str(Features), *cellIp)
-
-        #Uninstall everywhere
-        main.log.step( "Cleaning Enviornment..." )
-        for i in range(1, maxNodes + 1):
-            main.log.info(" Uninstalling ONOS " + str(i) )
-            main.ONOSbench.onosUninstall( ONOSIp[i] )
-        
-        #mvn clean install, for debugging set param 'skipCleanInstall' to yes to speed up test
-        if skipMvn != "yes":
-            mvnResult = main.ONOSbench.cleanInstall()
-                        
             #git
             main.step( "Git checkout and pull " + checkoutBranch )
             if gitPull == 'on':
@@ -83,52 +66,57 @@
                 checkoutResult = main.TRUE
                 pullResult = main.TRUE
                 main.log.info( "Skipped git checkout and pull" )
+        
+        # -- END OF INIT SECTION --#
+         
+        clusterCount = int(scale[0])
+        scale.remove(scale[0])       
+        
+        #kill off all onos processes 
+        main.log.step("Safety check, killing all ONOS processes")
+        main.log.step("before initiating enviornment setup")
+        for node in range(1, maxNodes + 1):
+            main.ONOSbench.onosDie(ONOSIp[node])
+        
+        #Uninstall everywhere
+        main.log.step( "Cleaning Enviornment..." )
+        for i in range(1, maxNodes + 1):
+            main.log.info(" Uninstalling ONOS " + str(i) )
+            main.ONOSbench.onosUninstall( ONOSIp[i] )
+       
+        #construct the cell file
+        main.log.info("Creating cell file")
+        cellIp = []
+        for node in range (1, clusterCount + 1):
+            cellIp.append(ONOSIp[node])
 
+        main.ONOSbench.createCellFile(BENCHIp,cellName,MN1Ip,str(Apps), *cellIp)
 
-        #main.step( "Set cell for ONOS cli env" )
-        #main.ONOS1cli.setCell( cellName )
         
         main.step( "Creating ONOS package" )
         packageResult = main.ONOSbench.onosPackage()  
 
-        main.step( "Installing ONOS package" )
-        install1Result = main.ONOSbench.onosInstall( node=ONOSIp[1] )
-
-        cellName = main.params[ 'ENV' ][ 'cellName' ]
-        main.step( "Applying cell file to environment" )
-        cellApplyResult = main.ONOSbench.setCell( cellName )
         main.step( "verify cells" )
         verifyCellResult = main.ONOSbench.verifyCell()
+       
+        for node in range(1, clusterCount + 1):
+            main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])
+            main.ONOSbench.onosInstall( ONOSIp[node])
 
-        main.step( "Set cell for ONOS cli env" )
-        cli1 = main.ONOS1cli.startOnosCli( ONOSIp[1] )
+        for node in range(1, clusterCount + 1):
+            for i in range( 2 ):
+                isup = main.ONOSbench.isup( ONOSIp[node] )
+                if isup:
+                    main.log.info("ONOS " + str(node) + " is up\n")
+                    break
+            if not isup:
+                main.log.report( "ONOS " + str(node) + " didn't start!" )
+        main.log.info("Startup sequence complete")
 
-        
     def CASE2( self, main ):
-        # This case increases the cluster size by whatever scale is
-        # Note: 'scale' is the size of the step
-        # if scaling is not a part of your test, simply run this case
-        # once after CASE1 to set up your enviornment for your desired 
-        # cluster size. If scaling is a part of you test call this case each time 
-        # you want to increase cluster size
-
-        ''                                                         
-        'Increase number of nodes and initiate CLI'
-        ''
-        import time
-        global clusterCount
-        
-        BENCHIp = main.params[ 'BENCH' ][ 'ip1' ]
-        scale = int( main.params[ 'SCALE' ] )
-        clusterCount += scale
-
-        main.log.report( "Increasing cluster size to " + str( clusterCount ) )
-        for node in range((clusterCount - scale) + 1, clusterCount + 1):
-            main.ONOSbench.onosDie(ONOSIp[node])
-            time.sleep(10)
-            main.log.info("Starting ONOS " + str(node) + " at IP: " + ONOSIp[node])    
-            main.ONOSbench.onosInstall( node=ONOSIp[node])
-            exec "a = main.ONOS%scli.startOnosCli" %str(node)
-            a(ONOSIp[node])
          
-
+        print ("clusterCount: " + str(clusterCount)) 
+        print ("scale: " + str(scale)) 
+        print ("ONOSIp: " + str(ONOSIp)) 
+        print ("INIT: " + str(init)) 
+        
diff --git a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.topo b/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.topo
index 8bd5a9f..0802eca 100644
--- a/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.topo
+++ b/TestON/tests/ScaleOutTemplate/ScaleOutTemplate.topo
@@ -66,7 +66,7 @@
         </ONOS3>
 
         <Mininet1>
-            <host>10.128.5.59</host>
+            <host>10.128.5.55</host>
             <user>admin</user>
             <password>onos_test</password>
             <type>MininetCliDriver</type>