diff --git a/TestON/drivers/common/cli/onosclidriver.py b/TestON/drivers/common/cli/onosclidriver.py
index 218b1e8..0e5e376 100755
--- a/TestON/drivers/common/cli/onosclidriver.py
+++ b/TestON/drivers/common/cli/onosclidriver.py
@@ -2376,9 +2376,11 @@
             # get total and installed number, see if they are match
             allState = response.get( 'all' )
             if allState.get( 'total' ) == allState.get( 'installed' ):
-                main.log.info( 'Total Intents: {}   Installed Intents: {}'.format( allState.get( 'total' ), allState.get( 'installed' ) ) )
+                main.log.info( 'Total Intents: {}   Installed Intents: {}'.format(
+                    allState.get( 'total' ), allState.get( 'installed' ) ) )
                 return main.TRUE
-            main.log.info( 'Verified Intents failed Excepte intetnes: {} installed intents: {}'.format( allState.get( 'total' ), allState.get( 'installed' ) ) )
+            main.log.info( 'Verified Intents failed Expected intents: {} installed intents: {}'.format(
+                allState.get( 'total' ), allState.get( 'installed' ) ) )
             return main.FALSE
 
         except ( TypeError, ValueError ):
@@ -5055,7 +5057,8 @@
                 logPaths = logPath + str( i ) + " " + logPaths
             cmd = "cat " + logPaths
             if startLine:
-                # 100000000 is just a extreme large number to make sure this function can grep all the lines after startLine
+                # 100000000 is just a extreme large number to make sure this function can
+                # grep all the lines after startLine
                 cmd = cmd + " | grep -A 100000000 \'" + startLine + "\'"
             if mode == 'all':
                 cmd = cmd + " | grep \'" + searchTerm + "\'"
@@ -5735,3 +5738,237 @@
         except Exception:
             main.log.exception( self.name + ": Uncaught exception!" )
             main.cleanAndExit()
+
+    def issu( self ):
+        """
+        Short summary of In-Service Software Upgrade status
+
+        Returns the output of the cli command or None on Error
+        """
+        try:
+            cmdStr = "issu"
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            return handle
+        except AssertionError:
+            main.log.exception( "" )
+            return None
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
+    def issuInit( self ):
+        """
+        Initiates an In-Service Software Upgrade
+
+        Returns main.TRUE on success, main.ERROR on error, else main.FALSE
+        """
+        try:
+            cmdStr = "issu init"
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            if "Initialized" in handle:
+                return main.TRUE
+            else:
+                return main.FALSE
+        except AssertionError:
+            main.log.exception( "" )
+            return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
+    def issuUpgrade( self ):
+        """
+        Transitions stores to upgraded nodes
+
+        Returns main.TRUE on success, main.ERROR on error, else main.FALSE
+        """
+        try:
+            cmdStr = "issu upgrade"
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            if "Upgraded" in handle:
+                return main.TRUE
+            else:
+                return main.FALSE
+        except AssertionError:
+            main.log.exception( "" )
+            return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
+    def issuCommit( self ):
+        """
+        Finalizes an In-Service Software Upgrade
+
+        Returns main.TRUE on success, main.ERROR on error, else main.FALSE
+        """
+        try:
+            cmdStr = "issu commit"
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            # TODO: Check the version returned by this command
+            if "Committed version" in handle:
+                return main.TRUE
+            else:
+                return main.FALSE
+        except AssertionError:
+            main.log.exception( "" )
+            return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
+    def issuRollback( self ):
+        """
+        Rolls back an In-Service Software Upgrade
+
+        Returns main.TRUE on success, main.ERROR on error, else main.FALSE
+        """
+        try:
+            cmdStr = "issu rollback"
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            # TODO: Check the version returned by this command
+            if "Rolled back to version" in handle:
+                return main.TRUE
+            else:
+                return main.FALSE
+        except AssertionError:
+            main.log.exception( "" )
+            return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
+    def issuReset( self ):
+        """
+        Resets the In-Service Software Upgrade status after a rollback
+
+        Returns main.TRUE on success, main.ERROR on error, else main.FALSE
+        """
+        try:
+            cmdStr = "issu reset"
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            # TODO: Check the version returned by this command
+            if "Reset version" in handle:
+                return main.TRUE
+            else:
+                return main.FALSE
+        except AssertionError:
+            main.log.exception( "" )
+            return main.ERROR
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return main.ERROR
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
+    def issuStatus( self ):
+        """
+        Status of an In-Service Software Upgrade
+
+        Returns the output of the cli command or None on Error
+        """
+        try:
+            cmdStr = "issu status"
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            return handle
+        except AssertionError:
+            main.log.exception( "" )
+            return None
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
+
+    def issuVersion( self ):
+        """
+        Get the version of an In-Service Software Upgrade
+
+        Returns the output of the cli command or None on Error
+        """
+        try:
+            cmdStr = "issu version"
+            handle = self.sendline( cmdStr )
+            assert handle is not None, "Error in sendline"
+            assert "Command not found:" not in handle, handle
+            assert "Unsupported command:" not in handle, handle
+            return handle
+        except AssertionError:
+            main.log.exception( "" )
+            return None
+        except TypeError:
+            main.log.exception( self.name + ": Object not as expected" )
+            return None
+        except pexpect.EOF:
+            main.log.error( self.name + ": EOF exception found" )
+            main.log.error( self.name + ":    " + self.handle.before )
+            main.cleanAndExit()
+        except Exception:
+            main.log.exception( self.name + ": Uncaught exception!" )
+            main.cleanAndExit()
diff --git a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
index 96ba015..2460c94 100644
--- a/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
+++ b/TestON/tests/HA/HAclusterRestart/HAclusterRestart.py
@@ -65,9 +65,6 @@
         start cli sessions
         start tcpdump
         """
-        import imp
-        import time
-        import json
         main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
                          "initialization" )
         # These are for csv plotting in jenkins
@@ -143,12 +140,6 @@
         except ( NameError, AttributeError ):
             main.log.error( "main.HAdata not defined, setting to []" )
             main.HAdata = []
-        # Reset non-persistent variables
-        try:
-            iCounterValue = 0
-        except NameError:
-            main.log.error( "iCounterValue not defined, setting to 0" )
-            iCounterValue = 0
 
         main.case( "Restart entire ONOS cluster" )
 
@@ -216,7 +207,7 @@
                                  " error logs" )
                 leaderResult = main.FALSE
             elif leaderN is None:
-                main.log.error( cli.name +
+                main.log.error( ctrl.name +
                                  " shows no leader for the election-app." )
                 leaderResult = main.FALSE
         if len( set( leaderList ) ) != 1:
diff --git a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.params b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.params
index a72a475..3d96b4c 100644
--- a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.params
+++ b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.params
@@ -21,6 +21,11 @@
     #CASE17: Check for basic functionality with distributed primitives
     <testcases>1,2,8,21,3,4,5,14,16,17,[61,8,7,4,15,17,62,7,8,4,15,17]*1000,8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
 
+    <GRAPH>
+        <nodeCluster>VM</nodeCluster>
+        <builds>20</builds>
+    </GRAPH>
+
     <apps></apps>
     <ONOS_Configuration>
         <org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
diff --git a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
index b7385fe..9a27fea 100644
--- a/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
+++ b/TestON/tests/HA/HAcontinuousStopNodes/HAcontinuousStopNodes.py
@@ -66,10 +66,6 @@
         start cli sessions
         start tcpdump
         """
-        import imp
-        import pexpect
-        import time
-        import json
         main.log.info( "ONOS HA test: Stop a minority of ONOS nodes - " +
                          "initialization" )
         # set global variables
@@ -140,7 +136,7 @@
         try:
             assert main.nodeIndex is not None, "main.nodeIndex not defined"
             assert main.killCount is not None, "main.killCount not defined"
-        except AttributeError as e:
+        except AttributeError:
             main.log.warn( "Node to kill not selected, defaulting to node 1" )
             main.nodeIndex = -1
             main.killCount = 1
@@ -211,9 +207,6 @@
         # Test of LeadershipElection
         leaderList = []
 
-        restarted = []
-        for ctrl in main.kill:
-            restarted.append( ctrl.ipAddress )
         leaderResult = main.TRUE
 
         for ctrl in main.Cluster.active():
@@ -230,11 +223,6 @@
                                  " shows no leader for the election-app was" +
                                  " elected after the old one died" )
                 leaderResult = main.FALSE
-            elif leaderN in restarted:
-                main.log.error( ctrl.name + " shows " + str( leaderN ) +
-                                 " as leader for the election-app, but it " +
-                                 "was restarted" )
-                leaderResult = main.FALSE
         if len( set( leaderList ) ) != 1:
             leaderResult = main.FALSE
             main.log.error(
diff --git a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
index 13424ed..37e106c 100644
--- a/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
+++ b/TestON/tests/HA/HAfullNetPartition/HAfullNetPartition.py
@@ -66,10 +66,6 @@
         start cli sessions
         start tcpdump
         """
-        import imp
-        import pexpect
-        import time
-        import json
         main.log.info( "ONOS HA test: Partition ONOS nodes into two sub-clusters - " +
                          "initialization" )
         # set global variables
@@ -134,7 +130,8 @@
         """
         The Failure case.
         """
-        import math
+        import pexpect
+        import time
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         main.case( "Partition ONOS nodes into two distinct partitions" )
@@ -163,37 +160,43 @@
             if i not in main.partition:
                 for j in main.partition:
                     foe = main.Cluster.runningNodes[ j ]
-                    main.log.warn( "Setting IP Tables rule from {} to {}. ".format( iCtrl.ipAddress, foe.ipAddress ) )
+                    main.log.warn( "Setting IP Tables rule from {} to {}. ".format( iCtrl.ipAddress,
+                                                                                    foe.ipAddress ) )
                     # CMD HERE
                     try:
-                        cmdStr = "sudo iptables -A {} -d {} -s {} -j DROP".format( "INPUT", iCtrl.ipAddress, foe.ipAddress )
+                        cmdStr = "sudo iptables -A {} -d {} -s {} -j DROP".format( "INPUT",
+                                                                                   iCtrl.ipAddress,
+                                                                                   foe.ipAddress )
                         this.sendline( cmdStr )
                         this.expect( "\$" )
                         main.log.debug( this.before )
                     except pexpect.EOF:
-                        main.log.error( self.name + ": EOF exception found" )
-                        main.log.error( self.name + ":    " + self.handle.before )
+                        main.log.error( iCtrl.name + ": EOF exception found" )
+                        main.log.error( iCtrl.name + ":    " + this.before )
                         main.cleanAndExit()
                     except Exception:
-                        main.log.exception( self.name + ": Uncaught exception!" )
+                        main.log.exception( iCtrl.name + ": Uncaught exception!" )
                         main.cleanAndExit()
             else:
                 for j in range( 0, n ):
                     if j not in main.partition:
                         foe = main.Cluster.runningNodes[ j ]
-                        main.log.warn( "Setting IP Tables rule from {} to {}. ".format( iCtrl.ipAddress, foe.ipAddress ) )
+                        main.log.warn( "Setting IP Tables rule from {} to {}. ".format( iCtrl.ipAddress,
+                                                                                        foe.ipAddress ) )
                         # CMD HERE
-                        cmdStr = "sudo iptables -A {} -d {} -s {} -j DROP".format( "INPUT", iCtrl.ipAddress, foe.ipAddress )
+                        cmdStr = "sudo iptables -A {} -d {} -s {} -j DROP".format( "INPUT",
+                                                                                   iCtrl.ipAddress,
+                                                                                   foe.ipAddress )
                         try:
                             this.sendline( cmdStr )
                             this.expect( "\$" )
                             main.log.debug( this.before )
                         except pexpect.EOF:
-                            main.log.error( self.name + ": EOF exception found" )
-                            main.log.error( self.name + ":    " + self.handle.before )
+                            main.log.error( iCtrl.name + ": EOF exception found" )
+                            main.log.error( iCtrl.name + ":    " + this.before )
                             main.cleanAndExit()
                         except Exception:
-                            main.log.exception( self.name + ": Uncaught exception!" )
+                            main.log.exception( iCtrl.name + ": Uncaught exception!" )
                             main.cleanAndExit()
                 main.Cluster.runningNodes[ i ].active = False
             iCtrl.Bench.exitFromSsh( this, iCtrl.ipAddress )
@@ -210,7 +213,6 @@
         """
         Healing Partition
         """
-        import time
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         assert main.partition, "main.partition not defined"
@@ -232,21 +234,11 @@
         for node in main.partition:
             main.Cluster.runningNodes[ node ].active = True
 
-        """
-        # NOTE : Not sure if this can be removed
-         main.activeNodes.sort()
-        try:
-            assert list( set( main.activeNodes ) ) == main.activeNodes,\
-                   "List of active nodes has duplicates, this likely indicates something was run out of order"
-        except AssertionError:
-            main.log.exception( "" )
-            main.cleanAndExit()
-        """
         main.step( "Checking ONOS nodes" )
         nodeResults = utilities.retry( main.Cluster.nodesCheck,
                                        False,
                                        sleep=15,
-                                       attempts=5 )
+                                       attempts=50 )
 
         utilities.assert_equals( expect=True, actual=nodeResults,
                                  onpass="Nodes check successful",
diff --git a/TestON/tests/HA/HAsanity/HAsanity.py b/TestON/tests/HA/HAsanity/HAsanity.py
index eb90c53..ee98b25 100644
--- a/TestON/tests/HA/HAsanity/HAsanity.py
+++ b/TestON/tests/HA/HAsanity/HAsanity.py
@@ -168,7 +168,7 @@
                                  " error logs" )
                 leaderResult = main.FALSE
             elif leaderN is None:
-                main.log.error( cli.name +
+                main.log.error( ctrl.name +
                                  " shows no leader for the election-app was" +
                                  " elected after the old one died" )
                 leaderResult = main.FALSE
diff --git a/TestON/tests/HA/HAscaling/HAscaling.py b/TestON/tests/HA/HAscaling/HAscaling.py
index d4b7b32..e862d1d 100644
--- a/TestON/tests/HA/HAscaling/HAscaling.py
+++ b/TestON/tests/HA/HAscaling/HAscaling.py
@@ -65,9 +65,6 @@
         start cli sessions
         start tcpdump
         """
-        import time
-        import os
-        import re
         main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
                          "initialization" )
         # set global variables
@@ -81,6 +78,7 @@
             main.log.error( "ONOSSetup not found. exiting the test" )
             main.cleanAndExit()
         main.testSetUp.envSetupDescription()
+        main.Cluster.numCtrls = 1
         try:
             from tests.HA.dependencies.HA import HA
             main.HA = HA()
@@ -260,7 +258,7 @@
                                  " error logs" )
                 leaderResult = main.FALSE
             elif leaderN is None:
-                main.log.error( cli.name +
+                main.log.error( ctrl.name +
                                  " shows no leader for the election-app." )
                 leaderResult = main.FALSE
         if len( set( leaderList ) ) != 1:
diff --git a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
index 74b2fc1..b18f8c4 100644
--- a/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
+++ b/TestON/tests/HA/HAsingleInstanceRestart/HAsingleInstanceRestart.py
@@ -65,9 +65,6 @@
         start cli sessions
         start tcpdump
         """
-        import imp
-        import time
-        import json
         main.log.info( "ONOS Single node cluster restart " +
                          "HA test - initialization" )
         main.case( "Setting up test environment" )
@@ -167,6 +164,11 @@
             main.log.warn( "ONOS1 intents response: " + repr( ONOSIntents ) )
         else:
             intentCheck = main.TRUE
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=intentCheck,
+            onpass="Intents are consistent across all ONOS nodes",
+            onfail="ONOS nodes have different views of intents" )
 
         main.step( "Get the flows from each controller" )
         global flowState
@@ -227,9 +229,6 @@
             main.log.exception( "Error parsing clusters[0]: " +
                                 repr( clusters[ 0 ] ) )
             numClusters = "ERROR"
-        clusterResults = main.FALSE
-        if numClusters == 1:
-            clusterResults = main.TRUE
         utilities.assert_equals(
             expect=1,
             actual=numClusters,
@@ -322,13 +321,6 @@
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
 
-        # Reset non-persistent variables
-        try:
-            iCounterValue = 0
-        except NameError:
-            main.log.error( "iCounterValue not defined, setting to 0" )
-            iCounterValue = 0
-
         main.case( "Restart ONOS node" )
         main.caseExplanation = "Killing ONOS process and restart cli " +\
                                 "sessions once onos is up."
@@ -556,7 +548,7 @@
                                 mnSwitches,
                                 json.loads( devices[ controller ] ),
                                 json.loads( ports[ controller ] ) )
-                    except ( TypeError, ValueError ) as e:
+                    except ( TypeError, ValueError ):
                         main.log.exception( "Object not as expected; devices={!r}\nports={!r}".format(
                             devices[ controller ], ports[ controller ] ) )
                 else:
diff --git a/TestON/tests/HA/HAswapNodes/HAswapNodes.py b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
index b221347..98a2e30 100644
--- a/TestON/tests/HA/HAswapNodes/HAswapNodes.py
+++ b/TestON/tests/HA/HAswapNodes/HAswapNodes.py
@@ -65,9 +65,6 @@
         start cli sessions
         start tcpdump
         """
-        import time
-        import os
-        import re
         main.log.info( "ONOS HA test: Restart all ONOS nodes - " +
                          "initialization" )
         # set global variables
@@ -139,7 +136,6 @@
         The Scaling case.
         """
         import time
-        import re
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         try:
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.params b/TestON/tests/HA/HAupgrade/HAupgrade.params
new file mode 100644
index 0000000..ba5d077
--- /dev/null
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.params
@@ -0,0 +1,98 @@
+<PARAMS>
+    #List of test cases:
+    #CASE1: Compile ONOS and push it to the test machines
+    #CASE2: Assign devices to controllers
+    #CASE21: Assign mastership to controllers
+    #CASE3: Assign intents
+    #CASE4: Ping across added host intents
+    #CASE5: Reading state of ONOS
+    #CASE61: The Failure inducing case.
+    #CASE62: The Failure recovery case.
+    #CASE7: Check state after control plane failure
+    #CASE8: Compare topo
+    #CASE9: Link s3-s28 down
+    #CASE10: Link s3-s28 up
+    #CASE11: Switch down
+    #CASE12: Switch up
+    #CASE13: Clean up
+    #CASE14: start election app on all onos nodes
+    #CASE15: Check that Leadership Election is still functional
+    #CASE16: Install Distributed Primitives app
+    #CASE17: Check for basic functionality with distributed primitives
+    <testcases>1,2,8,21,[3,4,5,14,16,17]*1,[60,8,61,8,62,8,63,8,64,8,7,4,15,17],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+
+    <GRAPH>
+        <nodeCluster>VM</nodeCluster>
+        <builds>20</builds>
+    </GRAPH>
+
+    <apps></apps>
+    <ONOS_Configuration>
+        <org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+            <useFlowObjectives>false</useFlowObjectives>
+            <defaultFlowObjectiveCompiler>org.onosproject.net.intent.impl.compiler.LinkCollectionIntentObjectiveCompiler</defaultFlowObjectiveCompiler>
+        </org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+    </ONOS_Configuration>
+    <ENV>
+        <cellName>HA</cellName>
+        <appString>drivers,openflow,proxyarp,mobility</appString>
+    </ENV>
+    <GIT>
+        <pull>False</pull>
+        <branch>master</branch>
+    </GIT>
+    <num_controllers> 7 </num_controllers>
+    <tcpdump> False </tcpdump>
+
+    <CTRL>
+        <port1>6653</port1>
+        <port2>6653</port2>
+        <port3>6653</port3>
+        <port4>6653</port4>
+        <port5>6653</port5>
+        <port6>6653</port6>
+        <port7>6653</port7>
+    </CTRL>
+    <BACKUP>
+        <ENABLED> False </ENABLED>
+        <TESTONUSER>sdn</TESTONUSER>
+        <TESTONIP>10.128.30.9</TESTONIP>
+    </BACKUP>
+    <PING>
+        <source1>h8</source1>
+        <source2>h9</source2>
+        <source3>h10</source3>
+        <source4>h11</source4>
+        <source5>h12</source5>
+        <source6>h13</source6>
+        <source7>h14</source7>
+        <source8>h15</source8>
+        <source9>h16</source9>
+        <source10>h17</source10>
+        <target1>10.0.0.18</target1>
+        <target2>10.0.0.19</target2>
+        <target3>10.0.0.20</target3>
+        <target4>10.0.0.21</target4>
+        <target5>10.0.0.22</target5>
+        <target6>10.0.0.23</target6>
+        <target7>10.0.0.24</target7>
+        <target8>10.0.0.25</target8>
+        <target9>10.0.0.26</target9>
+        <target10>10.0.0.27</target10>
+    </PING>
+    <timers>
+        <LinkDiscovery>12</LinkDiscovery>
+        <SwitchDiscovery>12</SwitchDiscovery>
+        <gossip>5</gossip>
+    </timers>
+    <kill>
+        <switch> s5 </switch>
+        <dpid> 0000000000005000 </dpid>
+        <links> h5 s2 s1 s6 </links>
+    </kill>
+    <MNtcpdump>
+        <intf>eth0</intf>
+        <port> </port>
+        <folder>~/packet_captures/</folder>
+    </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.py b/TestON/tests/HA/HAupgrade/HAupgrade.py
new file mode 100644
index 0000000..10e8fdc
--- /dev/null
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.py
@@ -0,0 +1,362 @@
+"""
+Copyright 2015 Open Networking Foundation (ONF)
+
+Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
+the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
+or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
+
+    TestON is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TestON is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with TestON.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+"""
+Description: This test is to determine if ONOS can handle
+    a minority of it's nodes restarting
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign devices to controllers
+CASE21: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE61: The Failure inducing case.
+CASE62: The Failure recovery case.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
+"""
+class HAupgrade:
+
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        """
+        CASE1 is to compile ONOS and push it to the test machines
+
+        Startup sequence:
+        cell <name>
+        onos-verify-cell
+        NOTE: temporary - onos-remove-raft-logs
+        onos-uninstall
+        start mininet
+        git pull
+        mvn clean install
+        onos-package
+        onos-install -f
+        onos-wait-for-start
+        start cli sessions
+        start tcpdump
+        """
+        main.log.info( "ONOS HA test: Stop a minority of ONOS nodes - " +
+                         "initialization" )
+        # These are for csv plotting in jenkins
+        main.HAlabels = []
+        main.HAdata = []
+        try:
+            from tests.dependencies.ONOSSetup import ONOSSetup
+            main.testSetUp = ONOSSetup()
+        except ImportError:
+            main.log.error( "ONOSSetup not found. exiting the test" )
+            main.cleanAndExit()
+        main.testSetUp.envSetupDescription()
+        try:
+            from tests.HA.dependencies.HA import HA
+            main.HA = HA()
+            cellName = main.params[ 'ENV' ][ 'cellName' ]
+            main.apps = main.params[ 'ENV' ][ 'appString' ]
+            stepResult = main.testSetUp.envSetup()
+        except Exception as e:
+            main.testSetUp.envSetupException( e )
+        main.testSetUp.evnSetupConclusion( stepResult )
+        main.HA.generateGraph( "HAupgrade" )
+
+        main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName=cellName, removeLog=True,
+                                  extraApply=[ main.HA.startingMininet,
+                                               main.HA.copyBackupConfig ],
+                                  extraClean=main.HA.cleanUpGenPartition )
+
+        main.HA.initialSetUp( serviceClean=True )
+
+    def CASE2( self, main ):
+        """
+        Assign devices to controllers
+        """
+        main.HA.assignDevices( main )
+
+    def CASE21( self, main ):
+        """
+        Assign mastership to controllers
+        """
+        main.HA.assignMastership( main )
+
+    def CASE3( self, main ):
+        """
+        Assign intents
+        """
+        main.HA.assignIntents( main )
+
+    def CASE4( self, main ):
+        """
+        Ping across added host intents
+        """
+        main.HA.pingAcrossHostIntent( main )
+
+    def CASE5( self, main ):
+        """
+        Reading state of ONOS
+        """
+        main.HA.readingState( main )
+
+    def CASE60( self, main ):
+        """
+        Initialize the upgrade.
+        """
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        main.case( "Initialize upgrade" )
+        main.HA.upgradeInit( main )
+
+    def CASE61( self, main ):
+        """
+        Upgrade a minority of nodes PHASE 1
+        """
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        main.case( "Upgrade minority of ONOS nodes" )
+
+        main.step( "Checking ONOS Logs for errors" )
+        for ctrl in main.Cluster.active():
+            main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
+            main.log.warn( ctrl.checkLogs( ctrl.ipAddress ) )
+
+        main.kill = []
+        n = len( main.Cluster.runningNodes )  # Number of nodes
+        p = n / 2  # Number of nodes in the minority
+        for i in range( p ):
+            main.kill.append( main.Cluster.runningNodes[ i ] )  # ONOS node to kill, listed by index in main.nodes
+        main.HA.upgradeNodes( main )
+
+        main.step( "Checking ONOS nodes" )
+        nodeResults = utilities.retry( main.Cluster.nodesCheck,
+                                       False,
+                                       sleep=15,
+                                       attempts=5 )
+        utilities.assert_equals( expect=True, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
+
+        if not nodeResults:
+            for ctrl in main.Cluster.active():
+                main.log.debug( "{} components not ACTIVE: \n{}".format(
+                    ctrl.name,
+                    ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
+            main.log.error( "Failed to start ONOS, stopping test" )
+            main.cleanAndExit()
+
+    def CASE62( self, main ):
+        """
+        Transfer to new version. PHASE 2
+        """
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        main.case( "Start the upgrade" )
+
+        main.step( "Send the command to switch to new version" )
+        ctrl = main.Cluster.next().CLI
+        upgraded = ctrl.issuUpgrade()
+        utilities.assert_equals( expect=main.TRUE, actual=upgraded,
+                                 onpass="Cluster has moved to the upgraded nodes",
+                                 onfail="Error transitioning to the upgraded nodes" )
+
+        main.step( "Check the status of the upgrade" )
+        ctrl = main.Cluster.next().CLI
+        status = ctrl.issu()
+        main.log.debug( status )
+        # TODO: check things here?
+
+        main.step( "Checking ONOS nodes" )
+        nodeResults = utilities.retry( main.Cluster.nodesCheck,
+                                       False,
+                                       sleep=15,
+                                       attempts=5 )
+        utilities.assert_equals( expect=True, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
+
+    def CASE63( self, main ):
+        """
+        Upgrade the rest of the nodes
+        """
+        main.case( "Upgrade remaining nodes" )
+        upgraded = main.kill
+        main.kill = []
+        for node in main.Cluster.runningNodes:
+            if node not in upgraded:
+                main.kill.append( node )
+
+        main.HA.upgradeNodes( main )
+
+    def CASE64( self, main ):
+        """
+        Commit to the upgrade.
+        """
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        main.case( "Commit upgrade" )
+
+        main.step( "Send the command to commit the upgrade" )
+        ctrl = main.Cluster.next().CLI
+        committed = ctrl.issuCommit()
+        utilities.assert_equals( expect=main.TRUE, actual=committed,
+                                 onpass="Upgrade has been committed",
+                                 onfail="Error committing the upgrade" )
+
+        main.step( "Check the status of the upgrade" )
+        ctrl = main.Cluster.next().CLI
+        status = ctrl.issu()
+        main.log.debug( status )
+        # TODO: check things here?
+
+    def CASE7( self, main ):
+        """
+        Check state after ONOS failure
+        """
+        try:
+            main.kill
+        except AttributeError:
+            main.kill = []
+
+        main.HA.checkStateAfterEvent( main, afterWhich=0 )
+        main.step( "Leadership Election is still functional" )
+        # Test of LeadershipElection
+        leaderList = []
+
+        restarted = []
+        for ctrl in main.kill:
+            restarted.append( ctrl.ipAddress )
+        leaderResult = main.TRUE
+
+        for ctrl in main.Cluster.active():
+            leaderN = ctrl.electionTestLeader()
+            leaderList.append( leaderN )
+            if leaderN == main.FALSE:
+                # error in response
+                main.log.error( "Something is wrong with " +
+                                 "electionTestLeader function, check the" +
+                                 " error logs" )
+                leaderResult = main.FALSE
+            elif leaderN is None:
+                main.log.error( ctrl.name +
+                                 " shows no leader for the election-app was" +
+                                 " elected after the old one died" )
+                leaderResult = main.FALSE
+            elif leaderN in restarted:
+                main.log.error( ctrl.name + " shows " + str( leaderN ) +
+                                 " as leader for the election-app, but it " +
+                                 "was restarted" )
+                leaderResult = main.FALSE
+        if len( set( leaderList ) ) != 1:
+            leaderResult = main.FALSE
+            main.log.error(
+                "Inconsistent view of leader for the election test app" )
+            # TODO: print the list
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=leaderResult,
+            onpass="Leadership election passed",
+            onfail="Something went wrong with Leadership election" )
+
+    def CASE8( self, main ):
+        """
+        Compare topo
+        """
+        main.HA.compareTopo( main )
+
+    def CASE9( self, main ):
+        """
+        Link s3-s28 down
+        """
+        main.HA.linkDown( main )
+
+    def CASE10( self, main ):
+        """
+        Link s3-s28 up
+        """
+        main.HA.linkUp( main )
+
+    def CASE11( self, main ):
+        """
+        Switch Down
+        """
+        # NOTE: You should probably run a topology check after this
+        main.HA.switchDown( main )
+
+    def CASE12( self, main ):
+        """
+        Switch Up
+        """
+        # NOTE: You should probably run a topology check after this
+        main.HA.switchUp( main )
+
+    def CASE13( self, main ):
+        """
+        Clean up
+        """
+        main.HAlabels.append( "Restart" )
+        main.HAdata.append( str( main.restartTime ) )
+        main.HA.cleanUp( main )
+
+    def CASE14( self, main ):
+        """
+        start election app on all onos nodes
+        """
+        main.HA.startElectionApp( main )
+
+    def CASE15( self, main ):
+        """
+        Check that Leadership Election is still functional
+            15.1 Run election on each node
+            15.2 Check that each node has the same leaders and candidates
+            15.3 Find current leader and withdraw
+            15.4 Check that a new node was elected leader
+            15.5 Check that that new leader was the candidate of old leader
+            15.6 Run for election on old leader
+            15.7 Check that oldLeader is a candidate, and leader if only 1 node
+            15.8 Make sure that the old leader was added to the candidate list
+
+            old and new variable prefixes refer to data from before vs after
+                withdrawl and later before withdrawl vs after re-election
+        """
+        main.HA.isElectionFunctional( main )
+
+    def CASE16( self, main ):
+        """
+        Install Distributed Primitives app
+        """
+        main.HA.installDistributedPrimitiveApp( main )
+
+    def CASE17( self, main ):
+        """
+        Check for basic functionality with distributed primitives
+        """
+        main.HA.checkDistPrimitivesFunc( main )
diff --git a/TestON/tests/HA/HAupgrade/HAupgrade.topo b/TestON/tests/HA/HAupgrade/HAupgrade.topo
new file mode 100644
index 0000000..4bf4bd4
--- /dev/null
+++ b/TestON/tests/HA/HAupgrade/HAupgrade.topo
@@ -0,0 +1,53 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOScell>
+            <host>localhost</host>  # ONOS "bench" machine
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosClusterDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS>
+                <cluster_name></cluster_name>  # Used as a prefix for cluster components. Defaults to 'ONOS'
+                <diff_clihost></diff_clihost>  # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
+                <karaf_username></karaf_username>
+                <karaf_password></karaf_password>
+                <web_user></web_user>
+                <web_pass></web_pass>
+                <rest_port></rest_port>
+                <prompt></prompt>  # TODO: we technically need a few of these, one per component
+                <onos_home></onos_home>  # defines where onos home is
+                <nodes> 7 </nodes>  # number of nodes in the cluster
+            </COMPONENTS>
+        </ONOScell>
+
+        <Mininet1>
+            <host>OCN</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>MininetCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS>
+                #Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/obelisk.py </arg1>
+                <arg2> --topo obelisk </arg2>
+                <arg3> --switch ovs,protocols=OpenFlow13 </arg3>
+                <controller> none </controller>
+                <home>~/mininet/custom/</home>
+                <prompt></prompt>
+            </COMPONENTS>
+        </Mininet1>
+
+        <Mininet2>
+            <host>OCN</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>RemoteMininetDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </Mininet2>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HA/HAupgrade/README b/TestON/tests/HA/HAupgrade/README
new file mode 100644
index 0000000..b3ffbfa
--- /dev/null
+++ b/TestON/tests/HA/HAupgrade/README
@@ -0,0 +1,29 @@
+This test is designed to verify that an ONOS cluster behaves correctly when some
+ONOS nodes are upgrade. Then test will initilize the upgrade then stop, upgrade,
+and restart a minority of the nodes in the cluster. Then we will start the first
+phase of the upgrade process to transfer to the new version. Then we will upgrade
+and restart the rest of the cluster. After that the test will verify everything
+works and commit the upgrade.
+
+As written, the test only supports an ONOS cluster of 3, 5, or 7 nodes.
+This is because the test doesn't apply to a single node cluster and ONOS clusters
+should be deployed in odd numbers.
+
+The gerneral structure for the test:
+- Startup
+- Assign switches
+- Verify ONOS state and functionality
+    - Device mastership
+    - Intents
+    - Leadership election
+    - Distributed Primitives
+- Initialize an upgrade
+- Upgrade some ONOS nodes
+- Verify ONOS state and functionality
+- Transfer to new version
+- Upgrade the rest of the nodes
+- Verify ONOS state and functionality
+- Commit the upgrade
+- Dataplane failures
+    - link down and up
+    - switch down and up
diff --git a/TestON/tests/HA/HAupgrade/__init__.py b/TestON/tests/HA/HAupgrade/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HA/HAupgrade/__init__.py
diff --git a/TestON/tests/HA/HAupgrade/dependencies/__init__.py b/TestON/tests/HA/HAupgrade/dependencies/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HA/HAupgrade/dependencies/__init__.py
diff --git a/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.params b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.params
new file mode 100644
index 0000000..ba5d077
--- /dev/null
+++ b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.params
@@ -0,0 +1,98 @@
+<PARAMS>
+    #List of test cases:
+    #CASE1: Compile ONOS and push it to the test machines
+    #CASE2: Assign devices to controllers
+    #CASE21: Assign mastership to controllers
+    #CASE3: Assign intents
+    #CASE4: Ping across added host intents
+    #CASE5: Reading state of ONOS
+    #CASE61: The Failure inducing case.
+    #CASE62: The Failure recovery case.
+    #CASE7: Check state after control plane failure
+    #CASE8: Compare topo
+    #CASE9: Link s3-s28 down
+    #CASE10: Link s3-s28 up
+    #CASE11: Switch down
+    #CASE12: Switch up
+    #CASE13: Clean up
+    #CASE14: start election app on all onos nodes
+    #CASE15: Check that Leadership Election is still functional
+    #CASE16: Install Distributed Primitives app
+    #CASE17: Check for basic functionality with distributed primitives
+    <testcases>1,2,8,21,[3,4,5,14,16,17]*1,[60,8,61,8,62,8,63,8,64,8,7,4,15,17],8,7,4,15,17,9,8,4,10,8,4,11,8,4,12,8,4,13</testcases>
+
+    <GRAPH>
+        <nodeCluster>VM</nodeCluster>
+        <builds>20</builds>
+    </GRAPH>
+
+    <apps></apps>
+    <ONOS_Configuration>
+        <org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+            <useFlowObjectives>false</useFlowObjectives>
+            <defaultFlowObjectiveCompiler>org.onosproject.net.intent.impl.compiler.LinkCollectionIntentObjectiveCompiler</defaultFlowObjectiveCompiler>
+        </org.onosproject.net.intent.impl.compiler.IntentConfigurableRegistrator>
+    </ONOS_Configuration>
+    <ENV>
+        <cellName>HA</cellName>
+        <appString>drivers,openflow,proxyarp,mobility</appString>
+    </ENV>
+    <GIT>
+        <pull>False</pull>
+        <branch>master</branch>
+    </GIT>
+    <num_controllers> 7 </num_controllers>
+    <tcpdump> False </tcpdump>
+
+    <CTRL>
+        <port1>6653</port1>
+        <port2>6653</port2>
+        <port3>6653</port3>
+        <port4>6653</port4>
+        <port5>6653</port5>
+        <port6>6653</port6>
+        <port7>6653</port7>
+    </CTRL>
+    <BACKUP>
+        <ENABLED> False </ENABLED>
+        <TESTONUSER>sdn</TESTONUSER>
+        <TESTONIP>10.128.30.9</TESTONIP>
+    </BACKUP>
+    <PING>
+        <source1>h8</source1>
+        <source2>h9</source2>
+        <source3>h10</source3>
+        <source4>h11</source4>
+        <source5>h12</source5>
+        <source6>h13</source6>
+        <source7>h14</source7>
+        <source8>h15</source8>
+        <source9>h16</source9>
+        <source10>h17</source10>
+        <target1>10.0.0.18</target1>
+        <target2>10.0.0.19</target2>
+        <target3>10.0.0.20</target3>
+        <target4>10.0.0.21</target4>
+        <target5>10.0.0.22</target5>
+        <target6>10.0.0.23</target6>
+        <target7>10.0.0.24</target7>
+        <target8>10.0.0.25</target8>
+        <target9>10.0.0.26</target9>
+        <target10>10.0.0.27</target10>
+    </PING>
+    <timers>
+        <LinkDiscovery>12</LinkDiscovery>
+        <SwitchDiscovery>12</SwitchDiscovery>
+        <gossip>5</gossip>
+    </timers>
+    <kill>
+        <switch> s5 </switch>
+        <dpid> 0000000000005000 </dpid>
+        <links> h5 s2 s1 s6 </links>
+    </kill>
+    <MNtcpdump>
+        <intf>eth0</intf>
+        <port> </port>
+        <folder>~/packet_captures/</folder>
+    </MNtcpdump>
+</PARAMS>
diff --git a/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
new file mode 100644
index 0000000..fd49fb1
--- /dev/null
+++ b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.py
@@ -0,0 +1,391 @@
+"""
+Copyright 2015 Open Networking Foundation (ONF)
+
+Please refer questions to either the onos test mailing list at <onos-test@onosproject.org>,
+the System Testing Plans and Results wiki page at <https://wiki.onosproject.org/x/voMg>,
+or the System Testing Guide page at <https://wiki.onosproject.org/x/WYQg>
+
+    TestON is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 2 of the License, or
+    (at your option) any later version.
+
+    TestON is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with TestON.  If not, see <http://www.gnu.org/licenses/>.
+"""
+
+"""
+Description: This test is to determine if ONOS can handle
+    a minority of it's nodes restarting
+
+List of test cases:
+CASE1: Compile ONOS and push it to the test machines
+CASE2: Assign devices to controllers
+CASE21: Assign mastership to controllers
+CASE3: Assign intents
+CASE4: Ping across added host intents
+CASE5: Reading state of ONOS
+CASE61: The Failure inducing case.
+CASE62: The Failure recovery case.
+CASE7: Check state after control plane failure
+CASE8: Compare topo
+CASE9: Link s3-s28 down
+CASE10: Link s3-s28 up
+CASE11: Switch down
+CASE12: Switch up
+CASE13: Clean up
+CASE14: start election app on all onos nodes
+CASE15: Check that Leadership Election is still functional
+CASE16: Install Distributed Primitives app
+CASE17: Check for basic functionality with distributed primitives
+"""
+class HAupgradeRollback:
+
+    def __init__( self ):
+        self.default = ''
+
+    def CASE1( self, main ):
+        """
+        CASE1 is to compile ONOS and push it to the test machines
+
+        Startup sequence:
+        cell <name>
+        onos-verify-cell
+        NOTE: temporary - onos-remove-raft-logs
+        onos-uninstall
+        start mininet
+        git pull
+        mvn clean install
+        onos-package
+        onos-install -f
+        onos-wait-for-start
+        start cli sessions
+        start tcpdump
+        """
+        main.log.info( "ONOS HA test: Stop a minority of ONOS nodes - " +
+                         "initialization" )
+        # These are for csv plotting in jenkins
+        main.HAlabels = []
+        main.HAdata = []
+        try:
+            from tests.dependencies.ONOSSetup import ONOSSetup
+            main.testSetUp = ONOSSetup()
+        except ImportError:
+            main.log.error( "ONOSSetup not found. exiting the test" )
+            main.cleanAndExit()
+        main.testSetUp.envSetupDescription()
+        try:
+            from tests.HA.dependencies.HA import HA
+            main.HA = HA()
+            cellName = main.params[ 'ENV' ][ 'cellName' ]
+            main.apps = main.params[ 'ENV' ][ 'appString' ]
+            stepResult = main.testSetUp.envSetup()
+        except Exception as e:
+            main.testSetUp.envSetupException( e )
+        main.testSetUp.evnSetupConclusion( stepResult )
+        main.HA.generateGraph( "HAupgrade" )
+
+        main.testSetUp.ONOSSetUp( main.Mininet1, main.Cluster, cellName=cellName, removeLog=True,
+                                  extraApply=[ main.HA.startingMininet,
+                                               main.HA.copyBackupConfig ],
+                                  extraClean=main.HA.cleanUpGenPartition )
+
+        main.HA.initialSetUp( serviceClean=True )
+
+    def CASE2( self, main ):
+        """
+        Assign devices to controllers
+        """
+        main.HA.assignDevices( main )
+
+    def CASE21( self, main ):
+        """
+        Assign mastership to controllers
+        """
+        main.HA.assignMastership( main )
+
+    def CASE3( self, main ):
+        """
+        Assign intents
+        """
+        main.HA.assignIntents( main )
+
+    def CASE4( self, main ):
+        """
+        Ping across added host intents
+        """
+        main.HA.pingAcrossHostIntent( main )
+
+    def CASE5( self, main ):
+        """
+        Reading state of ONOS
+        """
+        main.HA.readingState( main )
+
+    def CASE60( self, main ):
+        """
+        Initialize the upgrade.
+        """
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        main.case( "Initialize upgrade" )
+        main.HA.upgradeInit( main )
+
+    def CASE61( self, main ):
+        """
+        Upgrade a minority of nodes PHASE 1
+        """
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        main.case( "Upgrade minority of ONOS nodes" )
+
+        main.step( "Checking ONOS Logs for errors" )
+        for ctrl in main.Cluster.active():
+            main.log.debug( "Checking logs for errors on " + ctrl.name + ":" )
+            main.log.warn( ctrl.checkLogs( ctrl.ipAddress ) )
+
+        main.kill = []
+        n = len( main.Cluster.runningNodes )  # Number of nodes
+        p = n / 2  # Number of nodes in the minority
+        for i in range( p ):
+            main.kill.append( main.Cluster.runningNodes[ i ] )  # ONOS node to kill, listed by index in main.nodes
+        main.HA.upgradeNodes( main )
+
+        main.step( "Checking ONOS nodes" )
+        nodeResults = utilities.retry( main.Cluster.nodesCheck,
+                                       False,
+                                       sleep=15,
+                                       attempts=5 )
+        utilities.assert_equals( expect=True, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
+
+        if not nodeResults:
+            for ctrl in main.Cluster.active():
+                main.log.debug( "{} components not ACTIVE: \n{}".format(
+                    ctrl.name,
+                    ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
+            main.log.error( "Failed to start ONOS, stopping test" )
+            main.cleanAndExit()
+
+    def CASE62( self, main ):
+        """
+        Transfer to new version. PHASE 2
+        """
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        main.case( "Start the upgrade" )
+
+        main.step( "Send the command to switch to new version" )
+        ctrl = main.Cluster.next().CLI
+        upgraded = ctrl.issuUpgrade()
+        utilities.assert_equals( expect=main.TRUE, actual=upgraded,
+                                 onpass="Cluster has moved to the upgraded nodes",
+                                 onfail="Error transitioning to the upgraded nodes" )
+
+        main.step( "Check the status of the upgrade" )
+        ctrl = main.Cluster.next().CLI
+        status = ctrl.issu()
+        main.log.debug( status )
+        # TODO: check things here?
+
+        main.step( "Checking ONOS nodes" )
+        nodeResults = utilities.retry( main.Cluster.nodesCheck,
+                                       False,
+                                       sleep=15,
+                                       attempts=5 )
+        utilities.assert_equals( expect=True, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
+
+    def CASE63( self, main ):
+        """
+        Rollback the upgrade
+        """
+        main.case( "Rollback the upgrade" )
+        main.step( "Rollbak the upgrade" )
+        # send rollback command
+        ctrl = main.Cluster.next().CLI
+        rollback = ctrl.issuRollback()
+        utilities.assert_equals( expect=main.TRUE, actual=rollback,
+                                 onpass="Upgrade has been rolled back",
+                                 onfail="Error rolling back the upgrade" )
+
+        main.step( "Check the status of the upgrade" )
+        ctrl = main.Cluster.next().CLI
+        status = ctrl.issu()
+        main.log.debug( status )
+
+        # restart and reinstall old version on upgrade nodes
+        for ctrl in main.kill:
+            ctrl.onosStop( ctrl.ipAddress )
+            ctrl.onosUninstall( ctrl.ipAddress )
+            ctrl.onosInstall( options="-f", node=ctrl.ipAddress )
+            ctrl.onosSecureSSH( node=ctrl.ipAddress )
+            ctrl.startOnosCli( ctrl.ipAddress, waitForStart=True )
+        main.step( "Checking ONOS nodes" )
+        nodeResults = utilities.retry( main.Cluster.nodesCheck,
+                                       False,
+                                       sleep=15,
+                                       attempts=5 )
+        utilities.assert_equals( expect=True, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )
+
+        if not nodeResults:
+            for ctrl in main.Cluster.active():
+                main.log.debug( "{} components not ACTIVE: \n{}".format(
+                    ctrl.name,
+                    ctrl.CLI.sendline( "scr:list | grep -v ACTIVE" ) ) )
+            main.log.error( "Failed to start ONOS, stopping test" )
+            main.cleanAndExit()
+
+    def CASE64( self, main ):
+        """
+        Reset the upgrade state.
+        """
+        assert main, "main not defined"
+        assert utilities.assert_equals, "utilities.assert_equals not defined"
+        main.case( "Reset the upgrade state" )
+
+        main.step( "Send the command to reset the upgrade" )
+        ctrl = main.Cluster.next().CLI
+        committed = ctrl.issuCommit()
+        utilities.assert_equals( expect=main.TRUE, actual=committed,
+                                 onpass="Upgrade has been committed",
+                                 onfail="Error committing the upgrade" )
+
+        main.step( "Check the status of the upgrade" )
+        ctrl = main.Cluster.next().CLI
+        status = ctrl.issu()
+        main.log.debug( status )
+        # TODO: check things here?
+
+    def CASE7( self, main ):
+        """
+        Check state after ONOS failure
+        """
+        try:
+            main.kill
+        except AttributeError:
+            main.kill = []
+
+        main.HA.checkStateAfterEvent( main, afterWhich=0 )
+        main.step( "Leadership Election is still functional" )
+        # Test of LeadershipElection
+        leaderList = []
+
+        restarted = []
+        for ctrl in main.kill:
+            restarted.append( ctrl.ipAddress )
+        leaderResult = main.TRUE
+
+        for ctrl in main.Cluster.active():
+            leaderN = ctrl.electionTestLeader()
+            leaderList.append( leaderN )
+            if leaderN == main.FALSE:
+                # error in response
+                main.log.error( "Something is wrong with " +
+                                 "electionTestLeader function, check the" +
+                                 " error logs" )
+                leaderResult = main.FALSE
+            elif leaderN is None:
+                main.log.error( ctrl.name +
+                                 " shows no leader for the election-app was" +
+                                 " elected after the old one died" )
+                leaderResult = main.FALSE
+            elif leaderN in restarted:
+                main.log.error( ctrl.name + " shows " + str( leaderN ) +
+                                 " as leader for the election-app, but it " +
+                                 "was restarted" )
+                leaderResult = main.FALSE
+        if len( set( leaderList ) ) != 1:
+            leaderResult = main.FALSE
+            main.log.error(
+                "Inconsistent view of leader for the election test app" )
+            # TODO: print the list
+        utilities.assert_equals(
+            expect=main.TRUE,
+            actual=leaderResult,
+            onpass="Leadership election passed",
+            onfail="Something went wrong with Leadership election" )
+
+    def CASE8( self, main ):
+        """
+        Compare topo
+        """
+        main.HA.compareTopo( main )
+
+    def CASE9( self, main ):
+        """
+        Link s3-s28 down
+        """
+        main.HA.linkDown( main )
+
+    def CASE10( self, main ):
+        """
+        Link s3-s28 up
+        """
+        main.HA.linkUp( main )
+
+    def CASE11( self, main ):
+        """
+        Switch Down
+        """
+        # NOTE: You should probably run a topology check after this
+        main.HA.switchDown( main )
+
+    def CASE12( self, main ):
+        """
+        Switch Up
+        """
+        # NOTE: You should probably run a topology check after this
+        main.HA.switchUp( main )
+
+    def CASE13( self, main ):
+        """
+        Clean up
+        """
+        main.HAlabels.append( "Restart" )
+        main.HAdata.append( str( main.restartTime ) )
+        main.HA.cleanUp( main )
+
+    def CASE14( self, main ):
+        """
+        start election app on all onos nodes
+        """
+        main.HA.startElectionApp( main )
+
+    def CASE15( self, main ):
+        """
+        Check that Leadership Election is still functional
+            15.1 Run election on each node
+            15.2 Check that each node has the same leaders and candidates
+            15.3 Find current leader and withdraw
+            15.4 Check that a new node was elected leader
+            15.5 Check that that new leader was the candidate of old leader
+            15.6 Run for election on old leader
+            15.7 Check that oldLeader is a candidate, and leader if only 1 node
+            15.8 Make sure that the old leader was added to the candidate list
+
+            old and new variable prefixes refer to data from before vs after
+                withdrawl and later before withdrawl vs after re-election
+        """
+        main.HA.isElectionFunctional( main )
+
+    def CASE16( self, main ):
+        """
+        Install Distributed Primitives app
+        """
+        main.HA.installDistributedPrimitiveApp( main )
+
+    def CASE17( self, main ):
+        """
+        Check for basic functionality with distributed primitives
+        """
+        main.HA.checkDistPrimitivesFunc( main )
diff --git a/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.topo b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.topo
new file mode 100644
index 0000000..4bf4bd4
--- /dev/null
+++ b/TestON/tests/HA/HAupgradeRollback/HAupgradeRollback.topo
@@ -0,0 +1,53 @@
+<TOPOLOGY>
+    <COMPONENT>
+
+        <ONOScell>
+            <host>localhost</host>  # ONOS "bench" machine
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>OnosClusterDriver</type>
+            <connect_order>1</connect_order>
+            <COMPONENTS>
+                <cluster_name></cluster_name>  # Used as a prefix for cluster components. Defaults to 'ONOS'
+                <diff_clihost></diff_clihost>  # if it has different host other than localhost for CLI. True or empty. OC# will be used if True.
+                <karaf_username></karaf_username>
+                <karaf_password></karaf_password>
+                <web_user></web_user>
+                <web_pass></web_pass>
+                <rest_port></rest_port>
+                <prompt></prompt>  # TODO: we technically need a few of these, one per component
+                <onos_home></onos_home>  # defines where onos home is
+                <nodes> 7 </nodes>  # number of nodes in the cluster
+            </COMPONENTS>
+        </ONOScell>
+
+        <Mininet1>
+            <host>OCN</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>MininetCliDriver</type>
+            <connect_order>2</connect_order>
+            <COMPONENTS>
+                #Specify the Option for mininet
+                <arg1> --custom ~/mininet/custom/obelisk.py </arg1>
+                <arg2> --topo obelisk </arg2>
+                <arg3> --switch ovs,protocols=OpenFlow13 </arg3>
+                <controller> none </controller>
+                <home>~/mininet/custom/</home>
+                <prompt></prompt>
+            </COMPONENTS>
+        </Mininet1>
+
+        <Mininet2>
+            <host>OCN</host>
+            <user>sdn</user>
+            <password>rocks</password>
+            <type>RemoteMininetDriver</type>
+            <connect_order>3</connect_order>
+            <COMPONENTS>
+                <prompt></prompt>
+            </COMPONENTS>
+        </Mininet2>
+
+    </COMPONENT>
+</TOPOLOGY>
diff --git a/TestON/tests/HA/HAupgradeRollback/README b/TestON/tests/HA/HAupgradeRollback/README
new file mode 100644
index 0000000..396c222
--- /dev/null
+++ b/TestON/tests/HA/HAupgradeRollback/README
@@ -0,0 +1,29 @@
+This test is designed to verify that an ONOS cluster behaves correctly when some
+ONOS nodes are upgrade. Then test will initilize the upgrade then stop, upgrade,
+and restart a minority of the nodes in the cluster. Then we will start the first
+phase of the upgrade process to transfer to the new version. Then we will roll-back
+the upgrade, Restarted the upgraded nodes with the older version. After that the
+test will verify everything works and reset the upgrade.
+
+As written, the test only supports an ONOS cluster of 3, 5, or 7 nodes.
+This is because the test doesn't apply to a single node cluster and ONOS clusters
+should be deployed in odd numbers.
+
+The gerneral structure for the test:
+- Startup
+- Assign switches
+- Verify ONOS state and functionality
+    - Device mastership
+    - Intents
+    - Leadership election
+    - Distributed Primitives
+- Initialize an upgrade
+- Upgrade some ONOS nodes
+- Verify ONOS state and functionality
+- Transfer to new version
+- Upgrade the rest of the nodes
+- Verify ONOS state and functionality
+- Commit the upgrade
+- Dataplane failures
+    - link down and up
+    - switch down and up
diff --git a/TestON/tests/HA/HAupgradeRollback/__init__.py b/TestON/tests/HA/HAupgradeRollback/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HA/HAupgradeRollback/__init__.py
diff --git a/TestON/tests/HA/HAupgradeRollback/dependencies/__init__.py b/TestON/tests/HA/HAupgradeRollback/dependencies/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/TestON/tests/HA/HAupgradeRollback/dependencies/__init__.py
diff --git a/TestON/tests/HA/dependencies/HA.py b/TestON/tests/HA/dependencies/HA.py
index fa75618..c75ccdf 100644
--- a/TestON/tests/HA/dependencies/HA.py
+++ b/TestON/tests/HA/dependencies/HA.py
@@ -20,6 +20,8 @@
 """
 import json
 import time
+import pexpect
+import re
 
 
 class HA():
@@ -69,7 +71,6 @@
                                  onfail="Error starting Mininet" )
 
     def scalingMetadata( self ):
-        import re
         main.step( "Generate initial metadata file" )
         main.scaling = main.params[ 'scaling' ].split( "," )
         main.log.debug( main.scaling )
@@ -759,7 +760,6 @@
 
         pendingMap = main.Cluster.next().pendingMap()
         if not intentAddResult or "key" in pendingMap:
-            import time
             installedCheck = True
             main.log.info( "Sleeping 60 seconds to see if intents are found" )
             time.sleep( 60 )
@@ -1007,7 +1007,6 @@
         Reading state of ONOS
         """
         import json
-        import time
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
         try:
@@ -2477,8 +2476,6 @@
         """
         Clean up
         """
-        import os
-        import time
         assert main, "main not defined"
         assert utilities.assert_equals, "utilities.assert_equals not defined"
 
@@ -3763,3 +3760,29 @@
                                  onfail="Primitives app not activated" )
         # TODO check on all nodes instead of sleeping
         time.sleep( 5 )  # To allow all nodes to activate
+
+    def upgradeInit( self, main ):
+        '''
+        Initiates an update
+        '''
+        main.step( "Send the command to initialize the upgrade" )
+        ctrl = main.Cluster.next().CLI
+        initialized = ctrl.issuInit()
+        utilities.assert_equals( expect=main.TRUE, actual=initialized,
+                                 onpass="ISSU initialized",
+                                 onfail="Error initializing the upgrade" )
+
+        main.step( "Check the status of the upgrade" )
+        ctrl = main.Cluster.next().CLI
+        status = ctrl.issu()
+        main.log.debug( status )
+        # TODO: check things here?
+
+        main.step( "Checking ONOS nodes" )
+        nodeResults = utilities.retry( main.Cluster.nodesCheck,
+                                       False,
+                                       sleep=15,
+                                       attempts=5 )
+        utilities.assert_equals( expect=True, actual=nodeResults,
+                                 onpass="Nodes check successful",
+                                 onfail="Nodes check NOT successful" )