For ONOS-356

- Add current InstallationFuture information on TimeoutException
- Set timeout values propotional to batch size
- Fix for ConcurrentModificationException
- Check if src/dst is part of the graph before path computation

Change-Id: Iabeac7939c52502b83bf9ebcbe2023539de3ae99
diff --git a/core/net/src/main/java/org/onlab/onos/net/flow/impl/FlowRuleManager.java b/core/net/src/main/java/org/onlab/onos/net/flow/impl/FlowRuleManager.java
index 95bf8c5..3f37c29 100644
--- a/core/net/src/main/java/org/onlab/onos/net/flow/impl/FlowRuleManager.java
+++ b/core/net/src/main/java/org/onlab/onos/net/flow/impl/FlowRuleManager.java
@@ -67,7 +67,6 @@
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.atomic.AtomicReference;
-
 import static com.google.common.base.Preconditions.checkNotNull;
 import static org.onlab.util.Tools.namedThreads;
 import static org.slf4j.LoggerFactory.getLogger;
@@ -232,7 +231,7 @@
             lastSeen.remove(flowEntry);
             FlowEntry stored = store.getFlowEntry(flowEntry);
             if (stored == null) {
-                log.info("Rule already evicted from store: {}", flowEntry);
+                log.debug("Rule already evicted from store: {}", flowEntry);
                 return;
             }
             Device device = deviceService.getDevice(flowEntry.deviceId());
@@ -378,7 +377,8 @@
     // Store delegate to re-post events emitted from the store.
     private class InternalStoreDelegate implements FlowRuleStoreDelegate {
 
-        private static final int TIMEOUT = 5000; // ms
+        // FIXME set appropriate default and make it configurable
+        private static final int TIMEOUT_PER_OP = 500; // ms
 
         // TODO: Right now we only dispatch events at individual flowEntry level.
         // It may be more efficient for also dispatch events as a batch.
@@ -407,7 +407,7 @@
                     public void run() {
                         CompletedBatchOperation res;
                         try {
-                            res = result.get(TIMEOUT, TimeUnit.MILLISECONDS);
+                            res = result.get(TIMEOUT_PER_OP * batchOperation.size(), TimeUnit.MILLISECONDS);
                             store.batchOperationComplete(FlowRuleBatchEvent.completed(request, res));
                         } catch (TimeoutException | InterruptedException | ExecutionException e) {
                             log.warn("Something went wrong with the batch operation {}",
@@ -457,6 +457,10 @@
             if (state.get() == BatchState.FINISHED) {
                 return false;
             }
+            if (log.isDebugEnabled()) {
+                log.debug("Cancelling FlowRuleBatchFuture",
+                          new RuntimeException("Just printing backtrace"));
+            }
             if (!state.compareAndSet(BatchState.STARTED, BatchState.CANCELLED)) {
                 return false;
             }
@@ -526,6 +530,7 @@
                 throw new CancellationException();
             }
             if (!completed.isSuccess()) {
+                log.warn("FlowRuleBatch failed: {}", completed);
                 failed.addAll(completed.failedItems());
                 failedIds.addAll(completed.failedIds());
                 cleanUpBatch();
@@ -557,9 +562,11 @@
         }
 
         private void cleanUpBatch() {
+            log.debug("cleaning up batch");
+            // TODO convert these into a batch?
             for (FlowRuleBatchEntry fbe : batches.values()) {
                 if (fbe.getOperator() == FlowRuleOperation.ADD ||
-                        fbe.getOperator() == FlowRuleOperation.MODIFY) {
+                    fbe.getOperator() == FlowRuleOperation.MODIFY) {
                     store.deleteFlowRule(fbe.getTarget());
                 } else if (fbe.getOperator() == FlowRuleOperation.REMOVE) {
                     store.removeFlowRule(new DefaultFlowEntry(fbe.getTarget()));
diff --git a/core/net/src/main/java/org/onlab/onos/net/intent/impl/IntentManager.java b/core/net/src/main/java/org/onlab/onos/net/intent/impl/IntentManager.java
index 1cb3210..d93432d 100644
--- a/core/net/src/main/java/org/onlab/onos/net/intent/impl/IntentManager.java
+++ b/core/net/src/main/java/org/onlab/onos/net/intent/impl/IntentManager.java
@@ -734,7 +734,8 @@
 
     private class IntentInstallMonitor implements Runnable {
 
-        private static final long TIMEOUT = 5000; // ms
+        // TODO make this configurable
+        private static final int TIMEOUT_PER_OP = 500; // ms
         private static final int MAX_ATTEMPTS = 3;
 
         private final IntentOperations ops;
@@ -742,11 +743,18 @@
 
         private Future<CompletedBatchOperation> future;
         private long startTime = System.currentTimeMillis();
-        private long endTime = startTime + TIMEOUT;
+        private long endTime;
         private int installAttempt;
 
         public IntentInstallMonitor(IntentOperations ops) {
             this.ops = ops;
+            resetTimeoutLimit();
+        }
+
+        private void resetTimeoutLimit() {
+            // FIXME compute reasonable timeouts
+            this.endTime = System.currentTimeMillis()
+                           + ops.operations().size() * TIMEOUT_PER_OP;
         }
 
         private void buildIntentUpdates() {
@@ -805,6 +813,7 @@
          */
         private void processFutures() {
             if (future == null) {
+                log.warn("I have no Future.");
                 return; //FIXME look at this
             }
             try {
@@ -818,9 +827,10 @@
         }
 
         private void retry() {
+            log.debug("Execution timed out, retrying.");
             if (future.cancel(true)) { // cancel success; batch is reverted
                 // reset the timer
-                endTime = System.currentTimeMillis() + TIMEOUT;
+                resetTimeoutLimit();
                 if (installAttempt++ >= MAX_ATTEMPTS) {
                     log.warn("Install request timed out: {}", ops);
                     for (IntentUpdate update : intentUpdates) {
diff --git a/core/net/src/main/java/org/onlab/onos/net/intent/impl/ObjectiveTracker.java b/core/net/src/main/java/org/onlab/onos/net/intent/impl/ObjectiveTracker.java
index 7cc5d47..22673b6 100644
--- a/core/net/src/main/java/org/onlab/onos/net/intent/impl/ObjectiveTracker.java
+++ b/core/net/src/main/java/org/onlab/onos/net/intent/impl/ObjectiveTracker.java
@@ -169,8 +169,10 @@
                                         linkEvent.subject().isDurable())) {
                             final LinkKey linkKey = linkKey(linkEvent.subject());
                             Set<IntentId> intentIds = intentsByLink.get(linkKey);
-                            log.debug("recompile triggered by LinkDown {} {}", linkKey, intentIds);
-                            toBeRecompiled.addAll(intentIds);
+                            synchronized (intentsByLink) {
+                                log.debug("recompile triggered by LinkDown {} {}", linkKey, intentIds);
+                                toBeRecompiled.addAll(intentIds);
+                            }
                         }
                         recompileOnly = recompileOnly &&
                                 (linkEvent.type() == LINK_REMOVED ||