Dual-homing probing improvements

(1) Active probing mechanism in the following two scenarios
    (1-1) Probe all ports on the pair device within the same vlan (excluding the pair port) when the 1st location of a host is learnt
    (1-2) Probe again when a device/port goes down and comes up again
    * Introduce HostLocationProvingService
        - DISCOVER mode: discover potential new locations
        - VERIFY mode: verify old locations
    * Can be enabled/disabled via component config
    * Improve HostHandlerTest to test the probing behavior

(2) Fix an issue that redirection flow doesn't get installed after device re-connects

(3) Temporarily fix a race condition in HostHandler by adding a little bit delay

Change-Id: I33d3fe94a6ca491a88b8e06f65bef11447ead0bf
diff --git a/core/store/dist/src/main/java/org/onosproject/store/host/impl/DistributedHostStore.java b/core/store/dist/src/main/java/org/onosproject/store/host/impl/DistributedHostStore.java
index 436ec9b..dcf58e6 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/host/impl/DistributedHostStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/host/impl/DistributedHostStore.java
@@ -43,6 +43,7 @@
 import org.onosproject.net.host.HostEvent;
 import org.onosproject.net.host.HostStore;
 import org.onosproject.net.host.HostStoreDelegate;
+import org.onosproject.net.host.HostLocationProbingService.ProbeMode;
 import org.onosproject.net.provider.ProviderId;
 import org.onosproject.store.AbstractStore;
 import org.onosproject.store.serializers.KryoNamespaces;
@@ -60,6 +61,7 @@
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
@@ -146,7 +148,8 @@
 
         KryoNamespace.Builder pendingHostSerializer = KryoNamespace.newBuilder()
                 .register(KryoNamespaces.API)
-                .register(PendingHostLocation.class);
+                .register(PendingHostLocation.class)
+                .register(ProbeMode.class);
         pendingHostsConsistentMap = storageService.<MacAddress, PendingHostLocation>consistentMapBuilder()
                 .withName("onos-hosts-pending")
                 .withRelaxedReadConsistency()
@@ -311,7 +314,30 @@
     }
 
     @Override
+    public void appendLocation(HostId hostId, HostLocation location) {
+        log.debug("Appending location {} to host {}", location, hostId);
+        hosts.compute(hostId, (id, existingHost) -> {
+            if (existingHost != null) {
+                checkState(Objects.equals(hostId.mac(), existingHost.mac()),
+                        "Existing and new MAC addresses differ.");
+                checkState(Objects.equals(hostId.vlanId(), existingHost.vlan()),
+                        "Existing and new VLANs differ.");
+
+                Set<HostLocation> locations = new HashSet<>(existingHost.locations());
+                locations.add(location);
+
+                return new DefaultHost(existingHost.providerId(),
+                                hostId, existingHost.mac(), existingHost.vlan(),
+                                locations, existingHost.ipAddresses(),
+                                existingHost.configured(), existingHost.annotations());
+            }
+            return null;
+        });
+    }
+
+    @Override
     public void removeLocation(HostId hostId, HostLocation location) {
+        log.debug("Removing location {} from host {}", location, hostId);
         hosts.compute(hostId, (id, existingHost) -> {
             if (existingHost != null) {
                 checkState(Objects.equals(hostId.mac(), existingHost.mac()),
@@ -384,11 +410,11 @@
     }
 
     @Override
-    public MacAddress addPendingHostLocation(HostId hostId, HostLocation hostLocation) {
+    public MacAddress addPendingHostLocation(HostId hostId, ConnectPoint connectPoint, ProbeMode probeMode) {
         // Use ONLab OUI (3 bytes) + atomic counter (3 bytes) as the source MAC of the probe
         long nextIndex = storageService.getAtomicCounter("onos-hosts-probe-index").getAndIncrement();
         MacAddress probeMac = MacAddress.valueOf(MacAddress.NONE.toLong() + nextIndex);
-        PendingHostLocation phl = new PendingHostLocation(hostId, hostLocation);
+        PendingHostLocation phl = new PendingHostLocation(hostId, connectPoint, probeMode);
 
         pendingHostsCache.put(probeMac, phl);
         pendingHosts.put(probeMac, phl);
@@ -398,6 +424,14 @@
 
     @Override
     public void removePendingHostLocation(MacAddress probeMac) {
+        // Add the host location if probe replied in-time in DISCOVER mode
+        Optional.ofNullable(pendingHosts.get(probeMac)).ifPresent(phl -> {
+            if (phl.probeMode() == ProbeMode.DISCOVER) {
+                HostLocation newLocation = new HostLocation(phl.connectPoint(), System.currentTimeMillis());
+                appendLocation(phl.hostId(), newLocation);
+            }
+        });
+
         pendingHostsCache.invalidate(probeMac);
         pendingHosts.remove(probeMac);
     }
@@ -502,11 +536,13 @@
                 case INSERT:
                     break;
                 case UPDATE:
-                    if (newValue.value().expired()) {
+                    // Remove the host location if probe timeout in VERIFY mode
+                    if (newValue.value().expired() && newValue.value().probeMode() == ProbeMode.VERIFY) {
                         Executor locationRemover = Executors.newSingleThreadScheduledExecutor();
                         locationRemover.execute(() -> {
                             pendingHosts.remove(event.key());
-                            removeLocation(newValue.value().hostId(), newValue.value().location());
+                            removeLocation(newValue.value().hostId(),
+                                    new HostLocation(newValue.value().connectPoint(), 0L));
                         });
                     }
                     break;
diff --git a/core/store/dist/src/main/java/org/onosproject/store/host/impl/PendingHostLocation.java b/core/store/dist/src/main/java/org/onosproject/store/host/impl/PendingHostLocation.java
index de538f5..dc10507 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/host/impl/PendingHostLocation.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/host/impl/PendingHostLocation.java
@@ -16,8 +16,9 @@
 
 package org.onosproject.store.host.impl;
 
+import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.HostId;
-import org.onosproject.net.HostLocation;
+import org.onosproject.net.host.HostLocationProbingService.ProbeMode;
 
 import java.util.Objects;
 
@@ -28,19 +29,22 @@
  */
 class PendingHostLocation {
     private HostId hostId;
-    private HostLocation location;
+    private ConnectPoint connectPoint;
     private boolean expired;
+    private ProbeMode probeMode;
 
     /**
      * Constructs PendingHostLocation.
      *
      * @param hostId Host ID
-     * @param location location to be verified
+     * @param connectPoint location to be verified
+     * @param probeMode probe mode
      */
-    PendingHostLocation(HostId hostId, HostLocation location) {
+    PendingHostLocation(HostId hostId, ConnectPoint connectPoint, ProbeMode probeMode) {
         this.hostId = hostId;
-        this.location = location;
+        this.connectPoint = connectPoint;
         this.expired = false;
+        this.probeMode = probeMode;
     }
 
     /**
@@ -53,12 +57,12 @@
     }
 
     /**
-     * Gets HostLocation of this entry.
+     * Gets connect point of this entry.
      *
-     * @return host location
+     * @return connect point
      */
-    HostLocation location() {
-        return location;
+    ConnectPoint connectPoint() {
+        return connectPoint;
     }
 
     /**
@@ -79,6 +83,15 @@
         this.expired = expired;
     }
 
+    /**
+     * Gets probe mode of this entry.
+     *
+     * @return probe mode
+     */
+    ProbeMode probeMode() {
+        return probeMode;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -89,20 +102,22 @@
         }
         PendingHostLocation that = (PendingHostLocation) o;
         return (Objects.equals(this.hostId, that.hostId) &&
-                Objects.equals(this.location, that.location) &&
-                Objects.equals(this.expired, that.expired));
+                Objects.equals(this.connectPoint, that.connectPoint) &&
+                Objects.equals(this.expired, that.expired) &&
+                Objects.equals(this.probeMode, that.probeMode));
     }
     @Override
     public int hashCode() {
-        return Objects.hash(hostId, location, expired);
+        return Objects.hash(hostId, connectPoint, expired, probeMode);
     }
 
     @Override
     public String toString() {
         return toStringHelper(getClass())
                 .add("hostId", hostId)
-                .add("location", location)
+                .add("location", connectPoint)
                 .add("expired", expired)
+                .add("probeMode", probeMode)
                 .toString();
     }
 }