Dual-homing probing improvements

(1) Active probing mechanism in the following two scenarios
    (1-1) Probe all ports on the pair device within the same vlan (excluding the pair port) when the 1st location of a host is learnt
    (1-2) Probe again when a device/port goes down and comes up again
    * Introduce HostLocationProvingService
        - DISCOVER mode: discover potential new locations
        - VERIFY mode: verify old locations
    * Can be enabled/disabled via component config
    * Improve HostHandlerTest to test the probing behavior

(2) Fix an issue that redirection flow doesn't get installed after device re-connects

(3) Temporarily fix a race condition in HostHandler by adding a little bit delay

Change-Id: I33d3fe94a6ca491a88b8e06f65bef11447ead0bf
diff --git a/core/api/src/main/java/org/onosproject/net/host/HostLocationProbingService.java b/core/api/src/main/java/org/onosproject/net/host/HostLocationProbingService.java
new file mode 100644
index 0000000..56d056c
--- /dev/null
+++ b/core/api/src/main/java/org/onosproject/net/host/HostLocationProbingService.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2017-present Open Networking Foundation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.net.host;
+
+import org.onosproject.net.ConnectPoint;
+import org.onosproject.net.Host;
+
+public interface HostLocationProbingService {
+    /**
+     * Mode of host location probing.
+     */
+    enum ProbeMode {
+        /**
+         * Append probed host location if reply is received before timeout. Otherwise, do nothing.
+         * Typically used to discover secondary locations.
+         */
+        DISCOVER,
+
+        /**
+         * Remove probed host location if reply is received after timeout. Otherwise, do nothing.
+         * Typically used to verify previous locations.
+         */
+        VERIFY
+    }
+
+    /**
+     * Probes given host on given location.
+     *
+     * @param host the host to be probed
+     * @param connectPoint the location of host to be probed
+     * @param probeMode probe mode
+     */
+    void probeHostLocation(Host host, ConnectPoint connectPoint, ProbeMode probeMode);
+
+}
diff --git a/core/api/src/main/java/org/onosproject/net/host/HostProviderService.java b/core/api/src/main/java/org/onosproject/net/host/HostProviderService.java
index b324ad0..d9aeff5 100644
--- a/core/api/src/main/java/org/onosproject/net/host/HostProviderService.java
+++ b/core/api/src/main/java/org/onosproject/net/host/HostProviderService.java
@@ -17,8 +17,10 @@
 
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.MacAddress;
+import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.HostId;
 import org.onosproject.net.HostLocation;
+import org.onosproject.net.host.HostLocationProbingService.ProbeMode;
 import org.onosproject.net.provider.ProviderService;
 
 /**
@@ -64,10 +66,11 @@
      * retrieves the unique MAC address for the probe.
      *
      * @param hostId ID of the host
-     * @param hostLocation the host location that is under verification
+     * @param connectPoint the connect point that is under verification
+     * @param probeMode probe mode
      * @return probeMac, the source MAC address ONOS uses to probe the host
      */
-    default MacAddress addPendingHostLocation(HostId hostId, HostLocation hostLocation) {
+    default MacAddress addPendingHostLocation(HostId hostId, ConnectPoint connectPoint, ProbeMode probeMode) {
         return MacAddress.NONE;
     }
 
diff --git a/core/api/src/main/java/org/onosproject/net/host/HostStore.java b/core/api/src/main/java/org/onosproject/net/host/HostStore.java
index 6e98d95..96d3c31 100644
--- a/core/api/src/main/java/org/onosproject/net/host/HostStore.java
+++ b/core/api/src/main/java/org/onosproject/net/host/HostStore.java
@@ -23,6 +23,7 @@
 import org.onosproject.net.Host;
 import org.onosproject.net.HostId;
 import org.onosproject.net.HostLocation;
+import org.onosproject.net.host.HostLocationProbingService.ProbeMode;
 import org.onosproject.net.provider.ProviderId;
 import org.onosproject.store.Store;
 
@@ -65,6 +66,14 @@
     HostEvent removeIp(HostId hostId, IpAddress ipAddress);
 
     /**
+     * Append the specified location to the host entry.
+     *
+     * @param hostId host identification
+     * @param location location to be added
+     */
+    void appendLocation(HostId hostId, HostLocation location);
+
+    /**
      * Removes the specified location from the host entry.
      *
      * @param hostId host identification
@@ -139,10 +148,11 @@
      * retrieves the unique MAC address for the probe.
      *
      * @param hostId ID of the host
-     * @param hostLocation the host location that is under verification
+     * @param connectPoint the connect point that is under verification
+     * @param probeMode probe mode
      * @return probeMac, the source MAC address ONOS uses to probe the host
      */
-    default MacAddress addPendingHostLocation(HostId hostId, HostLocation hostLocation) {
+    default MacAddress addPendingHostLocation(HostId hostId, ConnectPoint connectPoint, ProbeMode probeMode) {
         return MacAddress.NONE;
     }
 
diff --git a/core/api/src/test/java/org/onosproject/net/host/HostStoreAdapter.java b/core/api/src/test/java/org/onosproject/net/host/HostStoreAdapter.java
index 8254de1..b79776d 100644
--- a/core/api/src/test/java/org/onosproject/net/host/HostStoreAdapter.java
+++ b/core/api/src/test/java/org/onosproject/net/host/HostStoreAdapter.java
@@ -65,6 +65,11 @@
     }
 
     @Override
+    public void appendLocation(HostId hostId, HostLocation location) {
+
+    }
+
+    @Override
     public void removeLocation(HostId hostId, HostLocation location) {
 
     }
diff --git a/core/common/src/test/java/org/onosproject/store/trivial/SimpleHostStore.java b/core/common/src/test/java/org/onosproject/store/trivial/SimpleHostStore.java
index e1fea28..4af7bea 100644
--- a/core/common/src/test/java/org/onosproject/store/trivial/SimpleHostStore.java
+++ b/core/common/src/test/java/org/onosproject/store/trivial/SimpleHostStore.java
@@ -174,6 +174,11 @@
     }
 
     @Override
+    public void appendLocation(HostId hostId, HostLocation location) {
+        hosts.get(hostId).locations().add(location);
+    }
+
+    @Override
     public void removeLocation(HostId hostId, HostLocation location) {
         hosts.get(hostId).locations().remove(location);
     }
diff --git a/core/net/src/main/java/org/onosproject/net/host/impl/HostManager.java b/core/net/src/main/java/org/onosproject/net/host/impl/HostManager.java
index b7a38e6..998f7df 100644
--- a/core/net/src/main/java/org/onosproject/net/host/impl/HostManager.java
+++ b/core/net/src/main/java/org/onosproject/net/host/impl/HostManager.java
@@ -29,6 +29,7 @@
 import org.onlab.packet.VlanId;
 import org.onlab.util.Tools;
 import org.onosproject.cfg.ComponentConfigService;
+import org.onosproject.net.host.HostLocationProbingService.ProbeMode;
 import org.onosproject.net.intf.Interface;
 import org.onosproject.net.intf.InterfaceService;
 import org.onosproject.net.HostLocation;
@@ -455,8 +456,8 @@
         }
 
         @Override
-        public MacAddress addPendingHostLocation(HostId hostId, HostLocation hostLocation) {
-            return store.addPendingHostLocation(hostId, hostLocation);
+        public MacAddress addPendingHostLocation(HostId hostId, ConnectPoint connectPoint, ProbeMode probeMode) {
+            return store.addPendingHostLocation(hostId, connectPoint, probeMode);
         }
 
         @Override
diff --git a/core/store/dist/src/main/java/org/onosproject/store/host/impl/DistributedHostStore.java b/core/store/dist/src/main/java/org/onosproject/store/host/impl/DistributedHostStore.java
index 436ec9b..dcf58e6 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/host/impl/DistributedHostStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/host/impl/DistributedHostStore.java
@@ -43,6 +43,7 @@
 import org.onosproject.net.host.HostEvent;
 import org.onosproject.net.host.HostStore;
 import org.onosproject.net.host.HostStoreDelegate;
+import org.onosproject.net.host.HostLocationProbingService.ProbeMode;
 import org.onosproject.net.provider.ProviderId;
 import org.onosproject.store.AbstractStore;
 import org.onosproject.store.serializers.KryoNamespaces;
@@ -60,6 +61,7 @@
 import java.util.Iterator;
 import java.util.Map;
 import java.util.Objects;
+import java.util.Optional;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
@@ -146,7 +148,8 @@
 
         KryoNamespace.Builder pendingHostSerializer = KryoNamespace.newBuilder()
                 .register(KryoNamespaces.API)
-                .register(PendingHostLocation.class);
+                .register(PendingHostLocation.class)
+                .register(ProbeMode.class);
         pendingHostsConsistentMap = storageService.<MacAddress, PendingHostLocation>consistentMapBuilder()
                 .withName("onos-hosts-pending")
                 .withRelaxedReadConsistency()
@@ -311,7 +314,30 @@
     }
 
     @Override
+    public void appendLocation(HostId hostId, HostLocation location) {
+        log.debug("Appending location {} to host {}", location, hostId);
+        hosts.compute(hostId, (id, existingHost) -> {
+            if (existingHost != null) {
+                checkState(Objects.equals(hostId.mac(), existingHost.mac()),
+                        "Existing and new MAC addresses differ.");
+                checkState(Objects.equals(hostId.vlanId(), existingHost.vlan()),
+                        "Existing and new VLANs differ.");
+
+                Set<HostLocation> locations = new HashSet<>(existingHost.locations());
+                locations.add(location);
+
+                return new DefaultHost(existingHost.providerId(),
+                                hostId, existingHost.mac(), existingHost.vlan(),
+                                locations, existingHost.ipAddresses(),
+                                existingHost.configured(), existingHost.annotations());
+            }
+            return null;
+        });
+    }
+
+    @Override
     public void removeLocation(HostId hostId, HostLocation location) {
+        log.debug("Removing location {} from host {}", location, hostId);
         hosts.compute(hostId, (id, existingHost) -> {
             if (existingHost != null) {
                 checkState(Objects.equals(hostId.mac(), existingHost.mac()),
@@ -384,11 +410,11 @@
     }
 
     @Override
-    public MacAddress addPendingHostLocation(HostId hostId, HostLocation hostLocation) {
+    public MacAddress addPendingHostLocation(HostId hostId, ConnectPoint connectPoint, ProbeMode probeMode) {
         // Use ONLab OUI (3 bytes) + atomic counter (3 bytes) as the source MAC of the probe
         long nextIndex = storageService.getAtomicCounter("onos-hosts-probe-index").getAndIncrement();
         MacAddress probeMac = MacAddress.valueOf(MacAddress.NONE.toLong() + nextIndex);
-        PendingHostLocation phl = new PendingHostLocation(hostId, hostLocation);
+        PendingHostLocation phl = new PendingHostLocation(hostId, connectPoint, probeMode);
 
         pendingHostsCache.put(probeMac, phl);
         pendingHosts.put(probeMac, phl);
@@ -398,6 +424,14 @@
 
     @Override
     public void removePendingHostLocation(MacAddress probeMac) {
+        // Add the host location if probe replied in-time in DISCOVER mode
+        Optional.ofNullable(pendingHosts.get(probeMac)).ifPresent(phl -> {
+            if (phl.probeMode() == ProbeMode.DISCOVER) {
+                HostLocation newLocation = new HostLocation(phl.connectPoint(), System.currentTimeMillis());
+                appendLocation(phl.hostId(), newLocation);
+            }
+        });
+
         pendingHostsCache.invalidate(probeMac);
         pendingHosts.remove(probeMac);
     }
@@ -502,11 +536,13 @@
                 case INSERT:
                     break;
                 case UPDATE:
-                    if (newValue.value().expired()) {
+                    // Remove the host location if probe timeout in VERIFY mode
+                    if (newValue.value().expired() && newValue.value().probeMode() == ProbeMode.VERIFY) {
                         Executor locationRemover = Executors.newSingleThreadScheduledExecutor();
                         locationRemover.execute(() -> {
                             pendingHosts.remove(event.key());
-                            removeLocation(newValue.value().hostId(), newValue.value().location());
+                            removeLocation(newValue.value().hostId(),
+                                    new HostLocation(newValue.value().connectPoint(), 0L));
                         });
                     }
                     break;
diff --git a/core/store/dist/src/main/java/org/onosproject/store/host/impl/PendingHostLocation.java b/core/store/dist/src/main/java/org/onosproject/store/host/impl/PendingHostLocation.java
index de538f5..dc10507 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/host/impl/PendingHostLocation.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/host/impl/PendingHostLocation.java
@@ -16,8 +16,9 @@
 
 package org.onosproject.store.host.impl;
 
+import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.HostId;
-import org.onosproject.net.HostLocation;
+import org.onosproject.net.host.HostLocationProbingService.ProbeMode;
 
 import java.util.Objects;
 
@@ -28,19 +29,22 @@
  */
 class PendingHostLocation {
     private HostId hostId;
-    private HostLocation location;
+    private ConnectPoint connectPoint;
     private boolean expired;
+    private ProbeMode probeMode;
 
     /**
      * Constructs PendingHostLocation.
      *
      * @param hostId Host ID
-     * @param location location to be verified
+     * @param connectPoint location to be verified
+     * @param probeMode probe mode
      */
-    PendingHostLocation(HostId hostId, HostLocation location) {
+    PendingHostLocation(HostId hostId, ConnectPoint connectPoint, ProbeMode probeMode) {
         this.hostId = hostId;
-        this.location = location;
+        this.connectPoint = connectPoint;
         this.expired = false;
+        this.probeMode = probeMode;
     }
 
     /**
@@ -53,12 +57,12 @@
     }
 
     /**
-     * Gets HostLocation of this entry.
+     * Gets connect point of this entry.
      *
-     * @return host location
+     * @return connect point
      */
-    HostLocation location() {
-        return location;
+    ConnectPoint connectPoint() {
+        return connectPoint;
     }
 
     /**
@@ -79,6 +83,15 @@
         this.expired = expired;
     }
 
+    /**
+     * Gets probe mode of this entry.
+     *
+     * @return probe mode
+     */
+    ProbeMode probeMode() {
+        return probeMode;
+    }
+
     @Override
     public boolean equals(Object o) {
         if (this == o) {
@@ -89,20 +102,22 @@
         }
         PendingHostLocation that = (PendingHostLocation) o;
         return (Objects.equals(this.hostId, that.hostId) &&
-                Objects.equals(this.location, that.location) &&
-                Objects.equals(this.expired, that.expired));
+                Objects.equals(this.connectPoint, that.connectPoint) &&
+                Objects.equals(this.expired, that.expired) &&
+                Objects.equals(this.probeMode, that.probeMode));
     }
     @Override
     public int hashCode() {
-        return Objects.hash(hostId, location, expired);
+        return Objects.hash(hostId, connectPoint, expired, probeMode);
     }
 
     @Override
     public String toString() {
         return toStringHelper(getClass())
                 .add("hostId", hostId)
-                .add("location", location)
+                .add("location", connectPoint)
                 .add("expired", expired)
+                .add("probeMode", probeMode)
                 .toString();
     }
 }