Increased number of retries and added variable delay to better withstand StorageException timeout.
Change-Id: Ic33d4e66e2787c717f7e3c90879d725a6e3e74d7
diff --git a/core/store/dist/src/main/java/org/onosproject/store/app/GossipApplicationStore.java b/core/store/dist/src/main/java/org/onosproject/store/app/GossipApplicationStore.java
index 936cf4e..b6c3b98 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/app/GossipApplicationStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/app/GossipApplicationStore.java
@@ -62,8 +62,8 @@
import static com.google.common.io.ByteStreams.toByteArray;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.onlab.util.Tools.delay;
import static org.onlab.util.Tools.groupedThreads;
+import static org.onlab.util.Tools.randomDelay;
import static org.onosproject.app.ApplicationEvent.Type.*;
import static org.onosproject.store.app.GossipApplicationStore.InternalState.*;
import static org.onosproject.store.service.EventuallyConsistentMapEvent.Type.PUT;
@@ -83,11 +83,10 @@
private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request");
- private static final int MAX_LOAD_RETRIES = 3;
+ private static final int MAX_LOAD_RETRIES = 5;
private static final int RETRY_DELAY_MS = 2_000;
private static final int FETCH_TIMEOUT_MS = 10_000;
- private static final int LOAD_TIMEOUT_MS = 5_000;
public enum InternalState {
INSTALLED, ACTIVATED, DEACTIVATED
@@ -169,7 +168,7 @@
}
} catch (Exception e) {
log.warn("Unable to load application {} from disk; retrying", name);
- delay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake
+ randomDelay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake
}
}
}
diff --git a/core/store/dist/src/main/java/org/onosproject/store/core/impl/ConsistentIdBlockStore.java b/core/store/dist/src/main/java/org/onosproject/store/core/impl/ConsistentIdBlockStore.java
index a421459..8924423 100644
--- a/core/store/dist/src/main/java/org/onosproject/store/core/impl/ConsistentIdBlockStore.java
+++ b/core/store/dist/src/main/java/org/onosproject/store/core/impl/ConsistentIdBlockStore.java
@@ -16,7 +16,7 @@
import java.util.Map;
-import static org.onlab.util.Tools.delay;
+import static org.onlab.util.Tools.randomDelay;
import static org.slf4j.LoggerFactory.getLogger;
/**
@@ -26,7 +26,7 @@
@Service
public class ConsistentIdBlockStore implements IdBlockStore {
- private static final int MAX_TRIES = 3;
+ private static final int MAX_TRIES = 5;
private static final int RETRY_DELAY_MS = 2_000;
private final Logger log = getLogger(getClass());
@@ -63,7 +63,7 @@
log.warn("Unable to allocate ID block due to {}; retrying...",
e.getMessage());
exc = e;
- delay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake
+ randomDelay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake
}
}
throw new IllegalStateException("Unable to allocate ID block", exc);
diff --git a/utils/misc/src/main/java/org/onlab/util/Tools.java b/utils/misc/src/main/java/org/onlab/util/Tools.java
index d9eb111..73c8802 100644
--- a/utils/misc/src/main/java/org/onlab/util/Tools.java
+++ b/utils/misc/src/main/java/org/onlab/util/Tools.java
@@ -39,6 +39,7 @@
import java.util.Collection;
import java.util.Dictionary;
import java.util.List;
+import java.util.Random;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
@@ -62,6 +63,8 @@
private static final Logger log = getLogger(Tools.class);
+ private static Random random = new Random();
+
/**
* Returns a thread factory that produces threads named according to the
* supplied name pattern.
@@ -195,6 +198,20 @@
}
/**
+ * Suspends the current thread for a random number of millis between 0 and
+ * the indicated limit.
+ *
+ * @param ms max number of millis
+ */
+ public static void randomDelay(int ms) {
+ try {
+ Thread.sleep(random.nextInt(ms));
+ } catch (InterruptedException e) {
+ throw new RuntimeException("Interrupted", e);
+ }
+ }
+
+ /**
* Suspends the current thread for a specified number of millis and nanos.
*
* @param ms number of millis