Phased recovery

- Implemented a set of CLI commands
    - Enable/disable group of ports
    - List recovery phase of each device
    - Force a specific device to enter given phase
- Return CompletableFuture in RRP
- Introduce completeAfter method in Tools
- Introduce submit method in PredictableExecutor which returns a CompletableFuture

Change-Id: I60b0fb7b67e392b33b52d908d2b53f7acbddc565
diff --git a/app/src/test/java/org/onosproject/segmentrouting/HostHandlerTest.java b/app/src/test/java/org/onosproject/segmentrouting/HostHandlerTest.java
index dedd66f..d67cb07 100644
--- a/app/src/test/java/org/onosproject/segmentrouting/HostHandlerTest.java
+++ b/app/src/test/java/org/onosproject/segmentrouting/HostHandlerTest.java
@@ -59,6 +59,7 @@
 
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.CompletableFuture;
 
 import static org.easymock.EasyMock.createMock;
 import static org.easymock.EasyMock.createNiceMock;
@@ -569,10 +570,12 @@
 
         // Host moved from [1A/1, 1B/1] to [1A/2, 1B/1]
         // We should expect only one bridging flow and one routing flow programmed on 1A
-        mockDefaultRoutingHandler.populateBridging(DEV3, P2, HOST_MAC, HOST_VLAN_UNTAGGED);
-        expectLastCall().times(1);
-        mockDefaultRoutingHandler.populateRoute(DEV3, HOST_IP11.toIpPrefix(), HOST_MAC, HOST_VLAN_UNTAGGED, P2, true);
-        expectLastCall().times(1);
+
+        expect(mockDefaultRoutingHandler.populateBridging(DEV3, P2, HOST_MAC, HOST_VLAN_UNTAGGED))
+                .andReturn(CompletableFuture.completedFuture(null)).once();
+        expect(mockDefaultRoutingHandler.populateRoute(DEV3, HOST_IP11.toIpPrefix(),
+                HOST_MAC, HOST_VLAN_UNTAGGED, P2, true))
+                .andReturn(CompletableFuture.completedFuture(null)).once();
         replay(mockDefaultRoutingHandler);
 
         hostHandler.processHostMovedEvent(new HostEvent(HostEvent.Type.HOST_MOVED, host2, host1));
diff --git a/app/src/test/java/org/onosproject/segmentrouting/MockFlowObjectiveService.java b/app/src/test/java/org/onosproject/segmentrouting/MockFlowObjectiveService.java
index 138b98a..0f2d7a9 100644
--- a/app/src/test/java/org/onosproject/segmentrouting/MockFlowObjectiveService.java
+++ b/app/src/test/java/org/onosproject/segmentrouting/MockFlowObjectiveService.java
@@ -31,6 +31,7 @@
 import org.onosproject.net.flowobjective.FlowObjectiveServiceAdapter;
 import org.onosproject.net.flowobjective.ForwardingObjective;
 import org.onosproject.net.flowobjective.Objective;
+import org.onosproject.net.flowobjective.ObjectiveError;
 
 import java.util.Map;
 
@@ -72,9 +73,13 @@
 
         if (op.equals(Objective.Operation.ADD)) {
             bridgingTable.put(btKey, btValue);
+            forwardingObjective.context().ifPresent(context -> context.onSuccess(forwardingObjective));
         } else if (op.equals(Objective.Operation.REMOVE)) {
             bridgingTable.remove(btKey, btValue);
+            forwardingObjective.context().ifPresent(context -> context.onSuccess(forwardingObjective));
         } else {
+            forwardingObjective.context().ifPresent(context ->
+                    context.onError(forwardingObjective, ObjectiveError.UNKNOWN));
             throw new IllegalArgumentException();
         }
     }
diff --git a/app/src/test/java/org/onosproject/segmentrouting/MockRoutingRulePopulator.java b/app/src/test/java/org/onosproject/segmentrouting/MockRoutingRulePopulator.java
index 77eeed2..4d222d8 100644
--- a/app/src/test/java/org/onosproject/segmentrouting/MockRoutingRulePopulator.java
+++ b/app/src/test/java/org/onosproject/segmentrouting/MockRoutingRulePopulator.java
@@ -21,8 +21,10 @@
 import org.onlab.packet.VlanId;
 import org.onosproject.net.DeviceId;
 import org.onosproject.net.PortNumber;
+import org.onosproject.net.flowobjective.Objective;
 
 import java.util.Map;
+import java.util.concurrent.CompletableFuture;
 
 /**
  * Mock Routing Rule Populator.
@@ -37,18 +39,20 @@
     }
 
     @Override
-    public void populateRoute(DeviceId deviceId, IpPrefix prefix,
-                              MacAddress hostMac, VlanId hostVlanId, PortNumber outPort, boolean directHost) {
+    public CompletableFuture<Objective> populateRoute(DeviceId deviceId, IpPrefix prefix, MacAddress hostMac,
+                                                      VlanId hostVlanId, PortNumber outPort, boolean directHost) {
         MockRoutingTableKey rtKey = new MockRoutingTableKey(deviceId, prefix);
         MockRoutingTableValue rtValue = new MockRoutingTableValue(outPort, hostMac, hostVlanId);
         routingTable.put(rtKey, rtValue);
+        return CompletableFuture.completedFuture(null);
     }
 
     @Override
-    public void revokeRoute(DeviceId deviceId, IpPrefix prefix,
+    public CompletableFuture<Objective> revokeRoute(DeviceId deviceId, IpPrefix prefix,
                             MacAddress hostMac, VlanId hostVlanId, PortNumber outPort, boolean directHost) {
         MockRoutingTableKey rtKey = new MockRoutingTableKey(deviceId, prefix);
         MockRoutingTableValue rtValue = new MockRoutingTableValue(outPort, hostMac, hostVlanId);
         routingTable.remove(rtKey, rtValue);
+        return CompletableFuture.completedFuture(null);
     }
 }
diff --git a/app/src/test/java/org/onosproject/segmentrouting/RouteHandlerTest.java b/app/src/test/java/org/onosproject/segmentrouting/RouteHandlerTest.java
index 6332856..f2a439b 100644
--- a/app/src/test/java/org/onosproject/segmentrouting/RouteHandlerTest.java
+++ b/app/src/test/java/org/onosproject/segmentrouting/RouteHandlerTest.java
@@ -39,6 +39,7 @@
 import org.onosproject.net.flow.TrafficTreatment;
 import org.onosproject.net.host.HostEvent;
 import org.onosproject.net.host.HostService;
+import org.onosproject.net.host.InterfaceIpAddress;
 import org.onosproject.net.intf.Interface;
 import org.onosproject.net.provider.ProviderId;
 import org.onosproject.routeservice.ResolvedRoute;
@@ -46,6 +47,7 @@
 import org.onosproject.routeservice.RouteEvent;
 import org.onosproject.segmentrouting.config.DeviceConfiguration;
 import org.onosproject.segmentrouting.config.SegmentRoutingDeviceConfig;
+import org.onosproject.segmentrouting.phasedrecovery.api.PhasedRecoveryService;
 import org.onosproject.store.service.StorageService;
 import org.onosproject.store.service.TestConsistentMap;
 import org.onosproject.store.service.TestConsistentMultimap;
@@ -82,15 +84,17 @@
     private static final IpPrefix P1 = IpPrefix.valueOf("10.0.0.0/24");
 
     // Single homed router 1
-    private static final IpAddress N1 = IpAddress.valueOf("10.0.1.254");
+    private static final IpAddress N1 = IpAddress.valueOf("10.0.1.1");
     private static final MacAddress M1 = MacAddress.valueOf("00:00:00:00:00:01");
     private static final VlanId V1 = VlanId.vlanId((short) 1);
     private static final ConnectPoint CP1 = ConnectPoint.deviceConnectPoint("of:0000000000000001/1");
     private static final Route R1 = new Route(Route.Source.STATIC, P1, N1);
     private static final ResolvedRoute RR1 = new ResolvedRoute(R1, M1, V1);
+    private static final Route DHCP_R1 = new Route(Route.Source.DHCP, P1, N1);
+    private static final ResolvedRoute DHCP_RR1 = new ResolvedRoute(DHCP_R1, M1, V1);
 
     // Single homed router 2
-    private static final IpAddress N2 = IpAddress.valueOf("10.0.2.254");
+    private static final IpAddress N2 = IpAddress.valueOf("10.0.2.1");
     private static final MacAddress M2 = MacAddress.valueOf("00:00:00:00:00:02");
     private static final VlanId V2 = VlanId.vlanId((short) 2);
     private static final ConnectPoint CP2 = ConnectPoint.deviceConnectPoint("of:0000000000000002/2");
@@ -98,14 +102,16 @@
     private static final ResolvedRoute RR2 = new ResolvedRoute(R2, M2, V2);
 
     // Dual homed router 1
-    private static final IpAddress N3 = IpAddress.valueOf("10.0.3.254");
+    private static final IpAddress N3 = IpAddress.valueOf("10.0.3.1");
     private static final MacAddress M3 = MacAddress.valueOf("00:00:00:00:00:03");
     private static final VlanId V3 = VlanId.vlanId((short) 3);
     private static final Route R3 = new Route(Route.Source.STATIC, P1, N3);
     private static final ResolvedRoute RR3 = new ResolvedRoute(R3, M3, V3);
+    private static final Route DHCP_R3 = new Route(Route.Source.DHCP, P1, N3);
+    private static final ResolvedRoute DHCP_RR3 = new ResolvedRoute(DHCP_R3, M3, V3);
 
     // Single homed router 3
-    private static final IpAddress N4 = IpAddress.valueOf("10.0.4.254");
+    private static final IpAddress N4 = IpAddress.valueOf("10.0.4.1");
     private static final MacAddress M4 = MacAddress.valueOf("00:00:00:00:00:04");
     private static final VlanId V4 = VlanId.vlanId((short) 4);
     private static final ConnectPoint CP4 = ConnectPoint.deviceConnectPoint("of:0000000000000004/4");
@@ -134,7 +140,15 @@
     // A set of devices of which we have mastership
     private static final Set<DeviceId> LOCAL_DEVICES = Sets.newHashSet(CP1.deviceId(), CP2.deviceId());
     // A set of interfaces
-    private static final Set<Interface> INTERFACES = Sets.newHashSet();
+    private static final InterfaceIpAddress IF_IP1 =
+            new InterfaceIpAddress(IpAddress.valueOf("10.0.1.254"), IpPrefix.valueOf("10.0.1.254/24"));
+    private static final InterfaceIpAddress IF_IP3 =
+            new InterfaceIpAddress(IpAddress.valueOf("10.0.3.254"), IpPrefix.valueOf("10.0.3.254/24"));
+    private static final Interface IF_CP1 = new Interface("if-cp1", CP1, Lists.newArrayList(IF_IP1, IF_IP3),
+            null, null, null, null, null);
+    private static final Interface IF_CP2 = new Interface("if-cp2", CP2, Lists.newArrayList(IF_IP1, IF_IP3),
+            null, null, null, null, null);
+    private static final Set<Interface> INTERFACES = Sets.newHashSet(IF_CP1, IF_CP2);
 
     @Before
     public void setUp() {
@@ -173,6 +187,9 @@
         srManager.hostService = hostService;
         srManager.cfgService = mockNetworkConfigRegistry;
         srManager.routeService = new MockRouteService(ROUTE_STORE);
+        srManager.phasedRecoveryService = createMock(PhasedRecoveryService.class);
+        expect(srManager.phasedRecoveryService.isEnabled()).andReturn(true).anyTimes();
+        replay(srManager.phasedRecoveryService);
 
         routeHandler = new RouteHandler(srManager);
 
@@ -194,7 +211,6 @@
         assertEquals(CP1.port(), rtv1.portNumber);
 
         assertEquals(1, SUBNET_TABLE.size());
-        assertTrue(SUBNET_TABLE.get(CP1).contains(P1));
     }
 
     @Test
@@ -214,8 +230,46 @@
         assertEquals(CP2.port(), rtv2.portNumber);
 
         assertEquals(2, SUBNET_TABLE.size());
-        assertTrue(SUBNET_TABLE.get(CP1).contains(P1));
-        assertTrue(SUBNET_TABLE.get(CP2).contains(P1));
+    }
+
+    // Only one of two dual-homed next hops present.
+    // Expect one routing table to be programmed with direct flow.
+    // The other is not programmed, not even for subnet
+    @Test
+    public void initDhcpRouteSingleDualHomeNextHop() {
+        ROUTE_STORE.put(P1, Sets.newHashSet(DHCP_RR1));
+
+        routeHandler.init(CP1.deviceId());
+
+        assertEquals(1, ROUTING_TABLE.size());
+        MockRoutingTableValue rtv1 = ROUTING_TABLE.get(new MockRoutingTableKey(CP1.deviceId(), P1));
+        assertEquals(M1, rtv1.macAddress);
+        assertEquals(V1, rtv1.vlanId);
+        assertEquals(CP1.port(), rtv1.portNumber);
+
+        assertEquals(1, SUBNET_TABLE.size());
+    }
+
+    // Both dual-homed next hops present.
+    // Expect both routing table to be programmed with direct flow
+    @Test
+    public void initDhcpRouteBothDualHomeNextHop() {
+        ROUTE_STORE.put(P1, Sets.newHashSet(DHCP_RR3));
+
+        routeHandler.init(CP1.deviceId());
+
+        assertEquals(2, ROUTING_TABLE.size());
+        MockRoutingTableValue rtv1 = ROUTING_TABLE.get(new MockRoutingTableKey(CP1.deviceId(), P1));
+        assertEquals(M3, rtv1.macAddress);
+        assertEquals(V3, rtv1.vlanId);
+        assertEquals(CP1.port(), rtv1.portNumber);
+
+        MockRoutingTableValue rtv2 = ROUTING_TABLE.get(new MockRoutingTableKey(CP2.deviceId(), P1));
+        assertEquals(M3, rtv2.macAddress);
+        assertEquals(V3, rtv2.vlanId);
+        assertEquals(CP2.port(), rtv2.portNumber);
+
+        assertEquals(2, SUBNET_TABLE.size());
     }
 
     @Test