[SDFAB-20] Prevent listeners ejection and the stop of the group polling

Offload listeners processing to external executors to prevent
the listener ejection due to time consuming processing

In future, we may want to extend the same fix to the
HostManager and NetworkConfigHostProvider

Additionally, avoid the propagation of the exceptions in GroupDriverProvider
which leads to the cancellation of the peridioc poll task

Change-Id: I8ea4ec9fda1ccc48bbd3855fd443ee8760cbbb60
diff --git a/apps/route-service/app/src/main/java/org/onosproject/routeservice/impl/RouteMonitor.java b/apps/route-service/app/src/main/java/org/onosproject/routeservice/impl/RouteMonitor.java
index ede60db..5ae2628 100644
--- a/apps/route-service/app/src/main/java/org/onosproject/routeservice/impl/RouteMonitor.java
+++ b/apps/route-service/app/src/main/java/org/onosproject/routeservice/impl/RouteMonitor.java
@@ -35,6 +35,7 @@
 
 import java.time.Duration;
 import java.util.Collection;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ScheduledExecutorService;
 import java.util.stream.Collectors;
 
@@ -65,6 +66,9 @@
     private final ScheduledExecutorService reaperExecutor =
             newSingleThreadScheduledExecutor(groupedThreads("route/reaper", "", log));
 
+    private final ExecutorService eventExecutor = newSingleThreadScheduledExecutor(groupedThreads(
+            "onos/routemonitor", "events-%d", log));
+
     /**
      * Creates a new route monitor.
      *
@@ -94,6 +98,8 @@
     public void shutdown() {
         stopProcessing();
         clusterService.removeListener(clusterListener);
+        eventExecutor.shutdownNow();
+        reaperExecutor.shutdownNow();
         asyncLock.unlock();
     }
 
@@ -145,31 +151,33 @@
 
         @Override
         public void event(ClusterEvent event) {
-            switch (event.type()) {
-            case INSTANCE_DEACTIVATED:
-                NodeId id = event.subject().id();
-                log.info("Node {} deactivated", id);
+            eventExecutor.execute(() -> {
+                switch (event.type()) {
+                    case INSTANCE_DEACTIVATED:
+                        NodeId id = event.subject().id();
+                        log.info("Node {} deactivated", id);
 
-                // DistributedLock is introduced to guarantee that minority nodes won't try to remove
-                // routes that originated from majority nodes.
-                // Adding 15 seconds retry for the leadership election to be completed.
-                asyncLock.tryLock(Duration.ofSeconds(15)).whenComplete((result, error) -> {
-                    if (result != null && result.isPresent()) {
-                        log.debug("Lock obtained. Put {} into removal queue", id);
-                        queue.addOne(id);
-                        asyncLock.unlock();
-                    } else {
-                        log.debug("Fail to obtain lock. Do not remove routes from {}", id);
-                    }
-                });
-                break;
-            case INSTANCE_ADDED:
-            case INSTANCE_REMOVED:
-            case INSTANCE_ACTIVATED:
-            case INSTANCE_READY:
-            default:
-                break;
-            }
+                        // DistributedLock is introduced to guarantee that minority nodes won't try to remove
+                        // routes that originated from majority nodes.
+                        // Adding 15 seconds retry for the leadership election to be completed.
+                        asyncLock.tryLock(Duration.ofSeconds(15)).whenComplete((result, error) -> {
+                            if (result != null && result.isPresent()) {
+                                log.debug("Lock obtained. Put {} into removal queue", id);
+                                queue.addOne(id);
+                                asyncLock.unlock();
+                            } else {
+                                log.debug("Fail to obtain lock. Do not remove routes from {}", id);
+                            }
+                        });
+                        break;
+                    case INSTANCE_ADDED:
+                    case INSTANCE_REMOVED:
+                    case INSTANCE_ACTIVATED:
+                    case INSTANCE_READY:
+                    default:
+                        break;
+                }
+            });
         }
     }
 }