Avoids disconnection from the switches

- Restarting the dispatcher during idle events
- Checking, during idle events, if the backlog can be drained
- Reducing the interval of idle events

Does not change the behavior of the state machine

Change-Id: I1721d8fad37e4e833d0fdfd12d51dc51a06559d0
diff --git a/protocols/openflow/ctl/src/main/java/org/onosproject/openflow/controller/impl/OFChannelHandler.java b/protocols/openflow/ctl/src/main/java/org/onosproject/openflow/controller/impl/OFChannelHandler.java
index b24ab80..a7a74f0 100644
--- a/protocols/openflow/ctl/src/main/java/org/onosproject/openflow/controller/impl/OFChannelHandler.java
+++ b/protocols/openflow/ctl/src/main/java/org/onosproject/openflow/controller/impl/OFChannelHandler.java
@@ -177,6 +177,12 @@
     @Reference(cardinality = ReferenceCardinality.MANDATORY)
     private OpenFlowService openFlowManager;
 
+    // Each IDLE_INTERVAL (see OFChannelInitializer) we will perform a sanity check
+    // Idle timeout actions will be performed each MAX_IDLE_RETRY * IDLE_INTERVAL
+    private static final int MAX_IDLE_RETRY = 4;
+    private int maxIdleRetry = MAX_IDLE_RETRY;
+
+    // Dispatcher buffer/read size
     private static final int BACKLOG_READ_BUFFER_DEFAULT = 1000;
 
     /**
@@ -1431,19 +1437,34 @@
         return getSwitchInfoString();
     }
 
+    // We have reduced the idle period, the idea is to use
+    // the IdleHandler to perform also some sanity checks.
+    // Previous code is still executed with the same frequency
+    // which is IDLE_INTERVAL * MAX_IDLE_RETRY of inactivity
     private void channelIdle(ChannelHandlerContext ctx,
                                IdleStateEvent e)
             throws IOException {
-        // Factory can be null if the channel goes idle during initial handshake. Since the switch
-        // is not even initialized properly, we just skip this and disconnect the channel.
-        if (factory != null) {
-            OFMessage m = factory.buildEchoRequest().build();
-            log.debug("Sending Echo Request on idle channel: {}", ctx.channel());
-            ctx.writeAndFlush(Collections.singletonList(m), ctx.voidPromise());
-            // XXX S some problems here -- echo request has no transaction id, and
-            // echo reply is not correlated to the echo request.
+        // dispatcher terminated for some reason, restart
+        if (dispatcherHandle.isDone()) {
+            dispatcherHandle = dispatcher.submit(new Dispatcher());
         }
-        state.processIdle(this);
+        // drain the backlog
+        processDispatchBacklogQueue();
+        // Original timeout reached
+        if (--maxIdleRetry == 0) {
+            maxIdleRetry = MAX_IDLE_RETRY;
+            // Factory can be null if the channel goes idle during initial handshake. Since the switch
+            // is not even initialized properly, we just skip this and disconnect the channel.
+            if (factory != null) {
+                // send an echo request each time idle_timeout * TICK
+                OFMessage m = factory.buildEchoRequest().build();
+                log.info("Sending Echo Request on idle channel: {}", ctx.channel());
+                // XXX S some problems here -- echo request has no transaction id, and
+                // echo reply is not correlated to the echo request.
+                ctx.writeAndFlush(Collections.singletonList(m), ctx.voidPromise());
+            }
+            state.processIdle(this);
+        }
     }
 
     @Override
@@ -1452,7 +1473,7 @@
             throws Exception {
         // If the connection is READER/WRITER idle try to send an echo request
         if (evt instanceof IdleStateEvent) {
-            log.info("Channel {} is {}", ctx.channel(), ((IdleStateEvent) evt).state());
+            log.debug("Channel {} is {}", ctx.channel(), ((IdleStateEvent) evt).state());
             channelIdle(ctx, (IdleStateEvent) evt);
         } else {
             super.userEventTriggered(ctx, evt);
@@ -1465,6 +1486,7 @@
                             Object msg) throws Exception {
 
         boolean release = true;
+        maxIdleRetry = MAX_IDLE_RETRY;
         try {
             if (msg instanceof OFMessage) {
                 // channelRead0 inlined
@@ -1618,29 +1640,34 @@
 
         if (dispatcherHandle.isDone()) {
             // dispatcher terminated for some reason, restart
-            dispatcherHandle = dispatcher.submit((Runnable) () -> {
-                try {
-                    for (;;) {
-                        int tc = 0;
-                        takeLock.lockInterruptibly();
-                        try {
-                            while ((tc = totalCount.get()) == 0) {
-                                notEmpty.await();
-                            }
-                        } finally {
-                            takeLock.unlock();
+            dispatcherHandle = dispatcher.submit(new Dispatcher());
+        }
+    }
+
+    private final class Dispatcher implements Runnable {
+        // dispatch loop
+        @Override
+        public void run() {
+            try {
+                for (;;) {
+                    int tc = 0;
+                    takeLock.lockInterruptibly();
+                    try {
+                        while ((tc = totalCount.get()) == 0) {
+                            notEmpty.await();
                         }
-
-                        processMessages(tc);
+                    } finally {
+                        takeLock.unlock();
                     }
-                } catch (InterruptedException e) {
-                    log.warn("Dispatcher interrupted");
-                    Thread.currentThread().interrupt();
-                    // interrupted. gracefully shutting down
-                    return;
-                }
 
-            });
+                    processMessages(tc);
+                }
+            } catch (InterruptedException e) {
+                log.warn("Dispatcher interrupted");
+                Thread.currentThread().interrupt();
+                // interrupted. gracefully shutting down
+                return;
+            }
         }
     }
 
diff --git a/protocols/openflow/ctl/src/main/java/org/onosproject/openflow/controller/impl/OFChannelInitializer.java b/protocols/openflow/ctl/src/main/java/org/onosproject/openflow/controller/impl/OFChannelInitializer.java
index a78e6a6..68cd614 100644
--- a/protocols/openflow/ctl/src/main/java/org/onosproject/openflow/controller/impl/OFChannelInitializer.java
+++ b/protocols/openflow/ctl/src/main/java/org/onosproject/openflow/controller/impl/OFChannelInitializer.java
@@ -79,7 +79,7 @@
 
         pipeline.addLast("consolidateflush", new FlushConsolidationHandler(
                            FlushConsolidationHandler.DEFAULT_EXPLICIT_FLUSH_AFTER_FLUSHES, true));
-        pipeline.addLast("idle", new IdleStateHandler(20, 25, 0));
+        pipeline.addLast("idle", new IdleStateHandler(5, 25, 0));
         pipeline.addLast("timeout", new ReadTimeoutHandler(30));
 
         // XXX S ONOS: was 15 increased it to fix Issue #296