Fix ConsistentMapException.Interrupted and NullPointerException

- Moving time-consuming packet processing to a separate thread
- Re-use the group information when dealing groupMissing instead of query again

Change-Id: I01f1b43260f22dcb969a105f16d04d79c722146e
diff --git a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
index 156240a..5b520b9 100644
--- a/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
+++ b/apps/segmentrouting/app/src/main/java/org/onosproject/segmentrouting/SegmentRoutingManager.java
@@ -146,6 +146,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.concurrent.TimeUnit;
@@ -278,6 +279,7 @@
     private ScheduledExecutorService hostEventExecutor;
     private ScheduledExecutorService routeEventExecutor;
     private ScheduledExecutorService mcastEventExecutor;
+    private ExecutorService packetExecutor;
 
     Map<DeviceId, DefaultGroupHandler> groupHandlerMap = new ConcurrentHashMap<>();
     /**
@@ -377,6 +379,7 @@
         hostEventExecutor = Executors.newSingleThreadScheduledExecutor(groupedThreads("sr-event-host", "%d", log));
         routeEventExecutor = Executors.newSingleThreadScheduledExecutor(groupedThreads("sr-event-route", "%d", log));
         mcastEventExecutor = Executors.newSingleThreadScheduledExecutor(groupedThreads("sr-event-mcast", "%d", log));
+        packetExecutor = Executors.newSingleThreadExecutor(groupedThreads("sr-packet", "%d", log));
 
         log.debug("Creating EC map nsnextobjectivestore");
         EventuallyConsistentMapBuilder<DestinationSetNextObjectiveStoreKey, NextNeighbors>
@@ -530,6 +533,7 @@
         hostEventExecutor.shutdown();
         routeEventExecutor.shutdown();
         mcastEventExecutor.shutdown();
+        packetExecutor.shutdown();
 
         cfgService.removeListener(cfgListener);
         cfgService.unregisterConfigFactory(deviceConfigFactory);
@@ -1036,7 +1040,10 @@
     private class InternalPacketProcessor implements PacketProcessor {
         @Override
         public void process(PacketContext context) {
+            packetExecutor.execute(() -> processPacketInternal(context));
+        }
 
+        private void processPacketInternal(PacketContext context) {
             if (context.isHandled()) {
                 return;
             }
diff --git a/core/net/src/main/java/org/onosproject/net/group/impl/GroupManager.java b/core/net/src/main/java/org/onosproject/net/group/impl/GroupManager.java
index c78108c..4a0e409 100644
--- a/core/net/src/main/java/org/onosproject/net/group/impl/GroupManager.java
+++ b/core/net/src/main/java/org/onosproject/net/group/impl/GroupManager.java
@@ -55,16 +55,17 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.Dictionary;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 import static com.google.common.base.Strings.isNullOrEmpty;
 import static org.onlab.util.Tools.get;
+import static org.onlab.util.Tools.groupedThreads;
 import static org.onosproject.security.AppGuard.checkPermission;
 import static org.onosproject.security.AppPermission.Type.GROUP_READ;
 import static org.onosproject.security.AppPermission.Type.GROUP_WRITE;
 import static org.slf4j.LoggerFactory.getLogger;
 
-
-
 /**
  * Provides implementation of the group service APIs.
  */
@@ -80,7 +81,9 @@
     private final GroupStoreDelegate delegate = new InternalGroupStoreDelegate();
     private final DeviceListener deviceListener = new InternalDeviceListener();
 
-    private final  GroupDriverProvider defaultProvider = new GroupDriverProvider();
+    private final GroupDriverProvider defaultProvider = new GroupDriverProvider();
+
+    private ExecutorService eventExecutor;
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected GroupStore store;
@@ -110,6 +113,7 @@
 
     @Activate
     public void activate(ComponentContext context) {
+        eventExecutor = Executors.newSingleThreadExecutor(groupedThreads("onos/group", "event"));
         store.setDelegate(delegate);
         eventDispatcher.addSink(GroupEvent.class, listenerRegistry);
         deviceService.addListener(deviceListener);
@@ -120,6 +124,7 @@
 
     @Deactivate
     public void deactivate() {
+        eventExecutor.shutdown();
         defaultProvider.terminate();
         deviceService.removeListener(deviceListener);
         cfgService.unregisterProperties(getClass(), false);
@@ -420,16 +425,19 @@
     }
 
     private class InternalDeviceListener implements DeviceListener {
-
         @Override
         public void event(DeviceEvent event) {
+            eventExecutor.execute(() -> processEventInternal(event));
+        }
+
+        private void processEventInternal(DeviceEvent event) {
             switch (event.type()) {
                 case DEVICE_REMOVED:
                 case DEVICE_AVAILABILITY_CHANGED:
                     DeviceId deviceId = event.subject().id();
                     if (!deviceService.isAvailable(deviceId)) {
-                        log.debug("Device {} became un available; clearing initial audit status",
-                                  event.type(), event.subject().id());
+                        log.debug("Device {} became unavailable; clearing initial audit status",
+                                event.type(), event.subject().id());
                         store.deviceInitialAuditCompleted(event.subject().id(), false);
 
                         if (purgeOnDisconnection) {
diff --git a/core/store/dist/src/main/java/org/onosproject/store/group/impl/DistributedGroupStore.java b/core/store/dist/src/main/java/org/onosproject/store/group/impl/DistributedGroupStore.java
index d8c3f7c..58417a8 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/group/impl/DistributedGroupStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/group/impl/DistributedGroupStore.java
@@ -1416,7 +1416,7 @@
                 }
             }
         }
-        for (Group group : storedGroupEntries) {
+        for (StoredGroupEntry group : storedGroupEntries) {
             // there are groups in the store that aren't in the switch
             log.debug("Group AUDIT: group {} missing in data plane for device {}",
                       group.id(), deviceId);
@@ -1470,7 +1470,7 @@
         return (group.referenceCount() == 0 && group.age() >= gcThresh);
     }
 
-    private void groupMissing(Group group) {
+    private void groupMissing(StoredGroupEntry group) {
         switch (group.state()) {
             case PENDING_DELETE:
                 log.debug("Group {} delete confirmation from device {}",
@@ -1481,19 +1481,13 @@
             case PENDING_ADD:
             case PENDING_ADD_RETRY:
             case PENDING_UPDATE:
-                log.debug("Group {} is in store but not on device {}",
-                          group, group.deviceId());
-                StoredGroupEntry existing =
-                        getStoredGroupEntry(group.deviceId(), group.id());
                 log.debug("groupMissing: group entry {} in device {} moving from {} to PENDING_ADD_RETRY",
-                          existing.id(),
-                          existing.deviceId(),
-                          existing.state());
-                existing.setState(Group.GroupState.PENDING_ADD_RETRY);
+                        group.id(),
+                        group.deviceId(),
+                        group.state());
+                group.setState(Group.GroupState.PENDING_ADD_RETRY);
                 //Re-PUT map entries to trigger map update events
-                getGroupStoreKeyMap().
-                        put(new GroupStoreKeyMapKey(existing.deviceId(),
-                                                    existing.appCookie()), existing);
+                getGroupStoreKeyMap().put(new GroupStoreKeyMapKey(group.deviceId(), group.appCookie()), group);
                 notifyDelegate(new GroupEvent(GroupEvent.Type.GROUP_ADD_REQUESTED,
                                               group));
                 break;
diff --git a/providers/host/src/main/java/org/onosproject/provider/host/impl/HostLocationProvider.java b/providers/host/src/main/java/org/onosproject/provider/host/impl/HostLocationProvider.java
index 54c3eee..dc58dcc 100644
--- a/providers/host/src/main/java/org/onosproject/provider/host/impl/HostLocationProvider.java
+++ b/providers/host/src/main/java/org/onosproject/provider/host/impl/HostLocationProvider.java
@@ -176,6 +176,7 @@
     private int probeInitDelayMs = 1000;
 
     ExecutorService eventHandler;
+    private ExecutorService packetHandler;
     private ScheduledExecutorService hostProber;
 
     /**
@@ -190,6 +191,8 @@
         cfgService.registerProperties(getClass());
         appId = coreService.registerApplication("org.onosproject.provider.host");
         eventHandler = newSingleThreadScheduledExecutor(groupedThreads("onos/host-loc-provider", "event-handler", log));
+        packetHandler = newSingleThreadScheduledExecutor(groupedThreads("onos/host-loc-provider",
+                "packet-handler", log));
         hostProber = newScheduledThreadPool(32, groupedThreads("onos/host-loc-probe", "%d", log));
         providerService = providerRegistry.register(this);
         packetService.addProcessor(processor, PacketProcessor.advisor(1));
@@ -210,6 +213,7 @@
         packetService.removeProcessor(processor);
         deviceService.removeListener(deviceListener);
         eventHandler.shutdown();
+        packetHandler.shutdown();
         hostProber.shutdown();
         providerService = null;
         log.info("Stopped");
@@ -510,6 +514,10 @@
 
         @Override
         public void process(PacketContext context) {
+            packetHandler.execute(() -> processPacketInternal(context));
+        }
+
+        private void processPacketInternal(PacketContext context) {
             if (context == null) {
                 return;
             }
diff --git a/providers/host/src/test/java/org/onosproject/provider/host/impl/HostLocationProviderTest.java b/providers/host/src/test/java/org/onosproject/provider/host/impl/HostLocationProviderTest.java
index f7b2f77..e00617d 100644
--- a/providers/host/src/test/java/org/onosproject/provider/host/impl/HostLocationProviderTest.java
+++ b/providers/host/src/test/java/org/onosproject/provider/host/impl/HostLocationProviderTest.java
@@ -22,6 +22,7 @@
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.onlab.junit.TestTools;
 import org.onlab.junit.TestUtils;
 import org.onlab.osgi.ComponentContextAdapter;
 import org.onlab.packet.ARP;
@@ -111,6 +112,8 @@
     private static final ProviderId PROVIDER_ID =
             new ProviderId("of", "org.onosproject.provider.host");
 
+    private static final int ASSERTION_DELAY = 100; // millis
+
     private static final Integer INPORT = 10;
     private static final Integer INPORT2 = 11;
     private static final String DEV1 = "of:1";
@@ -246,41 +249,51 @@
     public void events() {
         // New host. Expect one additional host description.
         testProcessor.process(new TestArpPacketContext(DEV1));
-        assertThat("New host expected", providerService.descriptions.size(), is(1));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("New host expected",
+                providerService.descriptions.size(), is(1)));
 
         // The host moved to new switch. Expect one additional host description.
         // The second host description should have a different location.
         testProcessor.process(new TestArpPacketContext(DEV2));
-        assertThat("Host motion expected", providerService.descriptions.size(), is(2));
-        HostLocation loc1 = providerService.descriptions.get(0).location();
-        HostLocation loc2 = providerService.descriptions.get(1).location();
-        assertNotEquals("Host location should be different", loc1, loc2);
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("Host motion expected",
+                providerService.descriptions.size(), is(2)));
+        final HostLocation loc11 = providerService.descriptions.get(0).location();
+        final HostLocation loc12 = providerService.descriptions.get(1).location();
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertNotEquals("Host location should be different",
+                loc11, loc12));
 
         // The host was misheard on a spine. Expect no additional host description.
         testProcessor.process(new TestArpPacketContext(DEV3));
-        assertThat("Host misheard on spine switch", providerService.descriptions.size(), is(2));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("Host misheard on spine switch",
+                providerService.descriptions.size(), is(2)));
 
         providerService.clear();
 
         // New host. Expect two additional host descriptions. One for target IP. One for dest IP.
         testProcessor.process(new TestNaPacketContext(DEV4));
-        assertThat("New host expected", providerService.descriptions.size(), is(2));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("New host expected",
+                providerService.descriptions.size(), is(2)));
 
         // The host moved to new switch. Expect two additional host descriptions.
         // The 3rd and 4th host description should have a different location.
         testProcessor.process(new TestNaPacketContext(DEV5));
-        assertThat("Host motion expected", providerService.descriptions.size(), is(4));
-        loc1 = providerService.descriptions.get(0).location();
-        loc2 = providerService.descriptions.get(1).location();
-        HostLocation loc3 = providerService.descriptions.get(2).location();
-        HostLocation loc4 = providerService.descriptions.get(3).location();
-        assertEquals("1st and 2nd location should be equal", loc1, loc2);
-        assertEquals("3rd and 4th location should be equal", loc3, loc4);
-        assertNotEquals("1st and 3rd location should be different", loc1, loc3);
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("Host motion expected",
+                providerService.descriptions.size(), is(4)));
+        final HostLocation loc21 = providerService.descriptions.get(0).location();
+        final HostLocation loc22 = providerService.descriptions.get(1).location();
+        final HostLocation loc23 = providerService.descriptions.get(2).location();
+        final HostLocation loc24 = providerService.descriptions.get(3).location();
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertEquals("1st and 2nd location should be equal",
+                loc21, loc22));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertEquals("3rd and 4th location should be equal",
+                loc23, loc24));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertNotEquals("1st and 3rd location should be different",
+                loc21, loc23));
 
         // The host was misheard on a spine. Expect no additional host description.
         testProcessor.process(new TestNaPacketContext(DEV6));
-        assertThat("Host misheard on spine switch", providerService.descriptions.size(), is(4));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("Host misheard on spine switch",
+                providerService.descriptions.size(), is(4)));
     }
 
     @Test
@@ -342,13 +355,13 @@
     @Test
     public void receiveArp() {
         testProcessor.process(new TestArpPacketContext(DEV1));
-        assertThat("receiveArp. One host description expected",
-                providerService.descriptions.size(), is(1));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveArp. One host description expected",
+                providerService.descriptions.size(), is(1)));
         HostDescription descr = providerService.descriptions.get(0);
-        assertThat(descr.location(), is(LOCATION));
-        assertThat(descr.hwAddress(), is(MAC));
-        assertThat(descr.ipAddress().toArray()[0], is(IP_ADDRESS));
-        assertThat(descr.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.location(), is(LOCATION)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.hwAddress(), is(MAC)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.ipAddress().toArray()[0], is(IP_ADDRESS)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.vlan(), is(VLAN)));
     }
 
     /**
@@ -357,13 +370,13 @@
     @Test
     public void receiveIpv4() {
         testProcessor.process(new TestIpv4PacketContext(DEV1));
-        assertThat("receiveIpv4. One host description expected",
-                providerService.descriptions.size(), is(1));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveIpv4. One host description expected",
+                providerService.descriptions.size(), is(1)));
         HostDescription descr = providerService.descriptions.get(0);
-        assertThat(descr.location(), is(LOCATION));
-        assertThat(descr.hwAddress(), is(MAC));
-        assertThat(descr.ipAddress().size(), is(0));
-        assertThat(descr.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.location(), is(LOCATION)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.hwAddress(), is(MAC)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.ipAddress().size(), is(0)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.vlan(), is(VLAN)));
     }
 
     /**
@@ -375,35 +388,36 @@
         TestUtils.setField(provider, "useDhcp", true);
         // DHCP Request
         testProcessor.process(new TestDhcpRequestPacketContext(DEV1, VLAN));
-        assertThat("receiveDhcpRequest. One host description expected",
-                providerService.descriptions.size(), is(1));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveDhcpRequest. One host description expected",
+                providerService.descriptions.size(), is(1)));
         // Should learn the MAC and location of DHCP client
         HostDescription descr = providerService.descriptions.get(0);
-        assertThat(descr.location(), is(LOCATION));
-        assertThat(descr.hwAddress(), is(MAC));
-        assertThat(descr.ipAddress().size(), is(0));
-        assertThat(descr.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.location(), is(LOCATION)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.hwAddress(), is(MAC)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.ipAddress().size(), is(0)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.vlan(), is(VLAN)));
 
         // DHCP Ack
         testProcessor.process(new TestDhcpAckPacketContext(DEV1));
-        assertThat("receiveDhcpAck. Two additional host descriptions expected",
-                providerService.descriptions.size(), is(3));
+        TestTools.assertAfter(ASSERTION_DELAY, () ->
+                assertThat("receiveDhcpAck. Two additional host descriptions expected",
+                providerService.descriptions.size(), is(3)));
 
         // Should also learn the MAC, location of DHCP server
         HostDescription descr2 = providerService.descriptions.get(1);
-        assertThat(descr2.location(), is(LOCATION3));
-        assertThat(descr2.hwAddress(), is(MAC3));
-        assertThat(descr2.ipAddress().size(), is(0));
-        assertThat(descr2.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr2.location(), is(LOCATION3)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr2.hwAddress(), is(MAC3)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr2.ipAddress().size(), is(0)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr2.vlan(), is(VLAN)));
 
         // Should update the IP address of the client.
         HostDescription descr3 = providerService.descriptions.get(2);
-        assertThat(descr3.location(), is(LOCATION));
-        assertThat(descr3.hwAddress(), is(MAC));
-        assertThat(descr3.ipAddress().size(), is(1));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr3.location(), is(LOCATION)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr3.hwAddress(), is(MAC)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr3.ipAddress().size(), is(1)));
         IpAddress ip = descr3.ipAddress().iterator().next();
-        assertThat(ip, is(IP_ADDRESS.getIp4Address()));
-        assertThat(descr3.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(ip, is(IP_ADDRESS.getIp4Address())));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr3.vlan(), is(VLAN)));
     }
 
     /**
@@ -415,35 +429,36 @@
         TestUtils.setField(provider, "useDhcp6", true);
         // DHCP Request
         testProcessor.process(new TestDhcp6RequestPacketContext(DEV4, VLAN));
-        assertThat("receiveDhcpRequest. One host description expected",
-                   providerService.descriptions.size(), is(1));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveDhcpRequest. One host description expected",
+                   providerService.descriptions.size(), is(1)));
         // Should learn the MAC and location of DHCP client
         HostDescription descr = providerService.descriptions.get(0);
-        assertThat(descr.location(), is(LOCATION2));
-        assertThat(descr.hwAddress(), is(MAC2));
-        assertThat(descr.ipAddress().size(), is(0));
-        assertThat(descr.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.location(), is(LOCATION2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.hwAddress(), is(MAC2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.ipAddress().size(), is(0)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.vlan(), is(VLAN)));
 
         // DHCP Ack
         testProcessor.process(new TestDhcp6AckPacketContext(DEV1));
-        assertThat("receiveDhcpAck. Two additional host descriptions expected",
-                   providerService.descriptions.size(), is(3));
+        TestTools.assertAfter(ASSERTION_DELAY, () ->
+                assertThat("receiveDhcpAck. Two additional host descriptions expected",
+                   providerService.descriptions.size(), is(3)));
 
         // Should also learn the MAC, location of DHCP server
         HostDescription descr2 = providerService.descriptions.get(1);
-        assertThat(descr2.location(), is(LOCATION3));
-        assertThat(descr2.hwAddress(), is(DHCP6_SERVER_MAC));
-        assertThat(descr2.ipAddress().size(), is(0));
-        assertThat(descr2.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr2.location(), is(LOCATION3)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr2.hwAddress(), is(DHCP6_SERVER_MAC)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr2.ipAddress().size(), is(0)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr2.vlan(), is(VLAN)));
 
         // Should update the IP address of the DHCP client.
         HostDescription descr3 = providerService.descriptions.get(2);
-        assertThat(descr3.location(), is(LOCATION2));
-        assertThat(descr3.hwAddress(), is(MAC2));
-        assertThat(descr3.ipAddress().size(), is(1));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr3.location(), is(LOCATION2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr3.hwAddress(), is(MAC2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr3.ipAddress().size(), is(1)));
         IpAddress ip = descr3.ipAddress().iterator().next();
-        assertThat(ip, is(IP_ADDRESS2.getIp6Address()));
-        assertThat(descr3.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(ip, is(IP_ADDRESS2.getIp6Address())));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr3.vlan(), is(VLAN)));
     }
 
     /**
@@ -453,19 +468,19 @@
     @Test
     public void receiveNa() {
         testProcessor.process(new TestNaPacketContext(DEV4));
-        assertThat("receiveNa. One host description expected",
-                providerService.descriptions.size(), is(2));
-        HostDescription descr = providerService.descriptions.get(0);
-        assertThat(descr.location(), is(LOCATION2));
-        assertThat(descr.hwAddress(), is(MAC2));
-        assertTrue(descr.ipAddress().contains(LLIP_ADDRESS2));
-        assertThat(descr.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveNa. One host description expected",
+                providerService.descriptions.size(), is(2)));
+        final HostDescription descr0 = providerService.descriptions.get(0);
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr0.location(), is(LOCATION2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr0.hwAddress(), is(MAC2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertTrue(descr0.ipAddress().contains(LLIP_ADDRESS2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr0.vlan(), is(VLAN)));
 
-        descr = providerService.descriptions.get(1);
-        assertThat(descr.location(), is(LOCATION2));
-        assertThat(descr.hwAddress(), is(MAC2));
-        assertTrue(descr.ipAddress().contains(IP_ADDRESS2));
-        assertThat(descr.vlan(), is(VLAN));
+        final HostDescription descr1 = providerService.descriptions.get(1);
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr1.location(), is(LOCATION2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr1.hwAddress(), is(MAC2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertTrue(descr1.ipAddress().contains(IP_ADDRESS2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr1.vlan(), is(VLAN)));
     }
 
     /**
@@ -474,13 +489,13 @@
     @Test
     public void receiveNs() {
         testProcessor.process(new TestNsPacketContext(DEV4));
-        assertThat("receiveNs. One host description expected",
-                providerService.descriptions.size(), is(1));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveNs. One host description expected",
+                providerService.descriptions.size(), is(1)));
         HostDescription descr = providerService.descriptions.get(0);
-        assertThat(descr.location(), is(LOCATION2));
-        assertThat(descr.hwAddress(), is(MAC2));
-        assertThat(descr.ipAddress().toArray()[0], is(IP_ADDRESS2));
-        assertThat(descr.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.location(), is(LOCATION2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.hwAddress(), is(MAC2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.ipAddress().toArray()[0], is(IP_ADDRESS2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.vlan(), is(VLAN)));
     }
 
     /**
@@ -489,8 +504,8 @@
     @Test
     public void receivesRa() {
         testProcessor.process(new TestRAPacketContext(DEV4));
-        assertThat("receivesRa. No host description expected",
-                providerService.descriptions.size(), is(0));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receivesRa. No host description expected",
+                providerService.descriptions.size(), is(0)));
     }
 
     /**
@@ -499,8 +514,8 @@
     @Test
     public void receiveRs() {
         testProcessor.process(new TestRSPacketContext(DEV4));
-        assertThat("receiveRs. No host description expected",
-                providerService.descriptions.size(), is(0));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveRs. No host description expected",
+                providerService.descriptions.size(), is(0)));
     }
 
     /**
@@ -509,8 +524,8 @@
     @Test
     public void receiveDad() {
         testProcessor.process(new TestDadPacketContext(DEV4));
-        assertThat("receiveDad. No host description expected",
-                providerService.descriptions.size(), is(0));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveDad. No host description expected",
+                providerService.descriptions.size(), is(0)));
     }
 
     /**
@@ -519,8 +534,8 @@
     @Test
     public void receiveIpv6Multicast() {
         testProcessor.process(new TestIpv6McastPacketContext(DEV4));
-        assertThat("receiveIpv6Multicast. No host description expected",
-                providerService.descriptions.size(), is(0));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveIpv6Multicast. No host description expected",
+                providerService.descriptions.size(), is(0)));
     }
 
     /**
@@ -529,13 +544,13 @@
     @Test
     public void receiveIpv6Unicast() {
         testProcessor.process(new TestIpv6PacketContext(DEV4));
-        assertThat("receiveIpv6Unicast. One host description expected",
-                providerService.descriptions.size(), is(1));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat("receiveIpv6Unicast. One host description expected",
+                providerService.descriptions.size(), is(1)));
         HostDescription descr = providerService.descriptions.get(0);
-        assertThat(descr.location(), is(LOCATION2));
-        assertThat(descr.hwAddress(), is(MAC2));
-        assertThat(descr.ipAddress().size(), is(0));
-        assertThat(descr.vlan(), is(VLAN));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.location(), is(LOCATION2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.hwAddress(), is(MAC2)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.ipAddress().size(), is(0)));
+        TestTools.assertAfter(ASSERTION_DELAY, () -> assertThat(descr.vlan(), is(VLAN)));
     }
 
     @After