[ONOS-7086] Detect failures in upgraded nodes and roll back upgrades

Change-Id: Ie804b5cd8cae0a1f0bbbe233dc34ae819c41673c
diff --git a/core/net/src/test/java/org/onosproject/upgrade/impl/UpgradeManagerTest.java b/core/net/src/test/java/org/onosproject/upgrade/impl/UpgradeManagerTest.java
index 4137744..48337cd 100644
--- a/core/net/src/test/java/org/onosproject/upgrade/impl/UpgradeManagerTest.java
+++ b/core/net/src/test/java/org/onosproject/upgrade/impl/UpgradeManagerTest.java
@@ -23,6 +23,7 @@
 
 import org.junit.Test;
 import org.onlab.packet.IpAddress;
+import org.onosproject.cluster.ClusterEvent;
 import org.onosproject.cluster.UnifiedClusterServiceAdapter;
 import org.onosproject.cluster.ControllerNode;
 import org.onosproject.cluster.DefaultControllerNode;
@@ -74,6 +75,15 @@
             }
 
             @Override
+            public ControllerNode getNode(NodeId nodeId) {
+                return getNodes()
+                        .stream()
+                        .filter(node -> node.id().equals(nodeId))
+                        .findFirst()
+                        .orElse(null);
+            }
+
+            @Override
             public ControllerNode.State getState(NodeId nodeId) {
                 return ControllerNode.State.READY;
             }
@@ -237,4 +247,22 @@
         assertEquals(Upgrade.Status.INACTIVE, upgradeManager.getState().status());
     }
 
+    @Test
+    public void testCrashRollback() throws Exception {
+        UpgradeManager upgradeManager = createUpgradeManager(
+                Version.version("1.0.0"),
+                new Upgrade(Version.version("1.0.0"), Version.version("1.0.1"), Upgrade.Status.UPGRADED),
+                Arrays.asList(Version.version("1.0.0"), Version.version("1.0.0"), Version.version("1.0.1")));
+
+        assertFalse(upgradeManager.isLocalActive());
+
+        upgradeManager.handleClusterEvent(new ClusterEvent(
+                ClusterEvent.Type.INSTANCE_DEACTIVATED,
+                upgradeManager.clusterService.getNode(NodeId.nodeId("2"))));
+
+        assertEquals(Upgrade.Status.ROLLED_BACK, upgradeManager.getState().status());
+        assertTrue(upgradeManager.isLocalActive());
+        assertFalse(upgradeManager.isLocalUpgraded());
+    }
+
 }