Handle link failure and receovery case for AVOID policy
diff --git a/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingManager.java b/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingManager.java
index 34961e5..8851420 100644
--- a/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingManager.java
+++ b/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingManager.java
@@ -275,8 +275,8 @@
             }
         });
 
-        testMode = TEST_MODE.POLICY_AVOID;
-        testTask.reschedule(30, TimeUnit.SECONDS);
+        //testMode = TEST_MODE.POLICY_AVOID;
+        //testTask.reschedule(30, TimeUnit.SECONDS);
     }
 
     @Override
@@ -758,9 +758,13 @@
                 }
             }
         }
+
+        UpdatePolicyRules();
+
         numOfPopulation++;
     }
 
+
     /**
      * populate the MPLS rules to handle Adjacency IDs
      *
@@ -1517,6 +1521,16 @@
         }
     }
 
+
+    private void UpdatePolicyRules() {
+
+        for (SegmentRoutingPolicy policy: policyTable.values()) {
+            if (policy.getType() == SegmentRoutingPolicy.PolicyType.AVOID) {
+                policy.updatePolicy();
+            }
+        }
+    }
+
     // ************************************
     // Utility functions
     // ************************************
@@ -2216,6 +2230,7 @@
             //nodesToAvoid.add(5);
             List<Link> linksToAvoid = new ArrayList<Link>();
 
+            /*
             Switch sw = mutableTopology.getSwitch(new Dpid(2));
             Link link = sw.getLinkToNeighbor(new Dpid(5));
             Switch sw2 = mutableTopology.getSwitch(new Dpid(4));
@@ -2225,12 +2240,17 @@
             linksToAvoid.add(link);
             linksToAvoid.add(link2);
             linksToAvoid.add(link3);
+            */
+
+            Switch sw = mutableTopology.getSwitch(new Dpid(5));
+            Link link = sw.getLinkToNeighbor(new Dpid(6));
+            linksToAvoid.add(link);
 
             createPolicy(pid, srcMac, dstMac, etherType, srcIp, dstIp, ipProto,
                     srcPort, dstPort, priority, srcNode, dstNode, nodesToAvoid, linksToAvoid);
 
-            testMode = TEST_MODE.POLICY_REMOVE3;
-            testTask.reschedule(10, TimeUnit.SECONDS);
+            //testMode = TEST_MODE.POLICY_REMOVE3;
+            //testTask.reschedule(10, TimeUnit.SECONDS);
 
         }
         else if (testMode == TEST_MODE.POLICY_REMOVE3) {
diff --git a/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingPolicy.java b/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingPolicy.java
index 1658b1c..7645bad 100644
--- a/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingPolicy.java
+++ b/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingPolicy.java
@@ -194,4 +194,11 @@
         return false;
     }
 
+    /**
+     * Update the policy rules if necessary according to the topology changes
+     */
+    public void updatePolicy() {
+
+    }
+
 }
diff --git a/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingPolicyAvoid.java b/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingPolicyAvoid.java
index bba968b..ff3d18c 100644
--- a/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingPolicyAvoid.java
+++ b/src/main/java/net/onrc/onos/apps/segmentrouting/SegmentRoutingPolicyAvoid.java
@@ -22,7 +22,7 @@
     private Switch dstSwitch;
     private List<String> dpidListToAvoid;
     private List<Link> linkListToAvoid;
-    private SegmentRoutingTunnel tunnel;
+    private List<SegmentRoutingTunnel> tunnels;
 
     public SegmentRoutingPolicyAvoid(PolicyNotification policyNotication) {
         super(policyNotication);
@@ -37,6 +37,7 @@
         this.dstSwitch = to;
         this.dpidListToAvoid = dpidList;
         this.linkListToAvoid = linksToAvoid;
+        this.tunnels = new ArrayList<SegmentRoutingTunnel>();
     }
 
     @Override
@@ -58,8 +59,9 @@
             labelStack.add(Integer.valueOf(srManager.getMplsLabel(dstDpid)));
             //String nodeToAvoid = srManager.getMplsLabel(switchToAvoid.getDpid().toString());
             OptimizeLabelStack(labelStack);
-            tunnel = new SegmentRoutingTunnel(
+            SegmentRoutingTunnel tunnel = new SegmentRoutingTunnel(
                     srManager, "avoid-0", labelStack);
+            tunnels.add(tunnel);
             if (tunnel.createTunnel()) {
                 //tunnelTable.put(tunnelId, srTunnel);
                 //TunnelNotification tunnelNotification =
@@ -78,16 +80,78 @@
 
     @Override
     public boolean removePolicy() {
-        if (tunnel.removeTunnel()) {
-            removeAclRules(tunnel.getRoutes());
-            return true;
+        for (SegmentRoutingTunnel tunnel: tunnels) {
+            if (tunnel.removeTunnel()) {
+                removeAclRules(tunnel.getRoutes());
+            }
+            else {
+                log.warn("Error in removing an avoid policy");
+                return false;
+            }
+        }
+
+        tunnels.removeAll(tunnels);
+        return true;
+    }
+
+    @Override
+    public void updatePolicy() {
+        ECMPShortestPathGraph graph = new ECMPShortestPathGraph(srcSwitch,
+                dpidListToAvoid, linkListToAvoid);
+        List<Path> ecmpPaths = graph.getECMPPaths(dstSwitch);
+
+        // Check if it needs update or not
+        boolean needUpdate = false;
+        for (Path path: ecmpPaths) {
+            List<Integer> labelStack = new ArrayList<Integer>();
+            for (int i=path.size()-1; i >=0; i--) {
+                LinkData link = path.get(i);
+                String dpid = link.getSrc().getDpid().toString();
+                labelStack.add(Integer.valueOf(srManager.getMplsLabel(dpid)));
+            }
+            String dstDpid = path.get(0).getDst().getDpid().toString();
+            labelStack.add(Integer.valueOf(srManager.getMplsLabel(dstDpid)));
+            OptimizeLabelStack(labelStack);
+            if (!checkIfIncluded(labelStack)) {
+                needUpdate = true;
+                break;
+            }
+        }
+
+        if (needUpdate) {
+            log.debug("Need to update the policy {}", policyId);
+            removePolicy();
+            createPolicy();
         }
         else {
-            log.warn("Error in removing an avoid policy");
-            return false;
+            log.debug("No need to update the policy {}, policyId");
         }
     }
 
+    private boolean checkIfIncluded(List<Integer> labelStack) {
+
+        for (SegmentRoutingTunnel tunnel: tunnels) {
+            if (tunnel.getLabelids().size() != labelStack.size())
+                continue;
+
+            int i = 0;
+            boolean identical = true;
+            for (Integer id: tunnel.getLabelids()) {
+                if (id != labelStack.get(i++)) {
+                    identical = false;
+                    break;
+                }
+            }
+            if (!identical)
+                continue;
+            else {
+                return true;
+            }
+        }
+
+        return false;
+    }
+
     /**
      * Optimize the label stack removing unnecessary label IDs, resulting the
      * same path. It modifies the list given directly.