Increased number of retries and added variable delay to better withstand StorageException timeout.

Change-Id: Ic33d4e66e2787c717f7e3c90879d725a6e3e74d7

@@ -62,8 +62,8 @@ import java.util.function.Function;
 
 
 import static com.google.common.io.ByteStreams.toByteArray;
 import static com.google.common.io.ByteStreams.toByteArray;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
 import static java.util.concurrent.TimeUnit.MILLISECONDS;
-import static org.onlab.util.Tools.delay;
 import static org.onlab.util.Tools.groupedThreads;
 import static org.onlab.util.Tools.groupedThreads;
+import static org.onlab.util.Tools.randomDelay;
 import static org.onosproject.app.ApplicationEvent.Type.*;
 import static org.onosproject.app.ApplicationEvent.Type.*;
 import static org.onosproject.store.app.GossipApplicationStore.InternalState.*;
 import static org.onosproject.store.app.GossipApplicationStore.InternalState.*;
 import static org.onosproject.store.service.EventuallyConsistentMapEvent.Type.PUT;
 import static org.onosproject.store.service.EventuallyConsistentMapEvent.Type.PUT;
@@ -83,11 +83,10 @@ public class GossipApplicationStore extends ApplicationArchive
 
 
     private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request");
     private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request");
 
 
-    private static final int MAX_LOAD_RETRIES = 3;
+    private static final int MAX_LOAD_RETRIES = 5;
     private static final int RETRY_DELAY_MS = 2_000;
     private static final int RETRY_DELAY_MS = 2_000;
 
 
     private static final int FETCH_TIMEOUT_MS = 10_000;
     private static final int FETCH_TIMEOUT_MS = 10_000;
-    private static final int LOAD_TIMEOUT_MS = 5_000;
 
 
     public enum InternalState {
     public enum InternalState {
         INSTALLED, ACTIVATED, DEACTIVATED
         INSTALLED, ACTIVATED, DEACTIVATED
@@ -169,7 +168,7 @@ public class GossipApplicationStore extends ApplicationArchive
                     }
                     }
                 } catch (Exception e) {
                 } catch (Exception e) {
                     log.warn("Unable to load application {} from disk; retrying", name);
                     log.warn("Unable to load application {} from disk; retrying", name);
-                    delay(RETRY_DELAY_MS);  // FIXME: This is a deliberate hack; fix in Drake
+                    randomDelay(RETRY_DELAY_MS);  // FIXME: This is a deliberate hack; fix in Drake
                 }
                 }
             }
             }
         }
         }

@@ -16,7 +16,7 @@ import org.slf4j.Logger;
 
 
 import java.util.Map;
 import java.util.Map;
 
 
-import static org.onlab.util.Tools.delay;
+import static org.onlab.util.Tools.randomDelay;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 
 
 /**
 /**
@@ -26,7 +26,7 @@ import static org.slf4j.LoggerFactory.getLogger;
 @Service
 @Service
 public class ConsistentIdBlockStore implements IdBlockStore {
 public class ConsistentIdBlockStore implements IdBlockStore {
 
 
-    private static final int MAX_TRIES = 3;
+    private static final int MAX_TRIES = 5;
     private static final int RETRY_DELAY_MS = 2_000;
     private static final int RETRY_DELAY_MS = 2_000;
 
 
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
@@ -63,7 +63,7 @@ public class ConsistentIdBlockStore implements IdBlockStore {
                 log.warn("Unable to allocate ID block due to {}; retrying...",
                 log.warn("Unable to allocate ID block due to {}; retrying...",
                          e.getMessage());
                          e.getMessage());
                 exc = e;
                 exc = e;
-                delay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake
+                randomDelay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake
             }
             }
         }
         }
         throw new IllegalStateException("Unable to allocate ID block", exc);
         throw new IllegalStateException("Unable to allocate ID block", exc);

@@ -39,6 +39,7 @@ import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collection;
 import java.util.Dictionary;
 import java.util.Dictionary;
 import java.util.List;
 import java.util.List;
+import java.util.Random;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.Future;
@@ -62,6 +63,8 @@ public abstract class Tools {
 
 
     private static final Logger log = getLogger(Tools.class);
     private static final Logger log = getLogger(Tools.class);
 
 
+    private static Random random = new Random();
+
     /**
     /**
      * Returns a thread factory that produces threads named according to the
      * Returns a thread factory that produces threads named according to the
      * supplied name pattern.
      * supplied name pattern.
@@ -195,6 +198,20 @@ public abstract class Tools {
     }
     }
 
 
     /**
     /**
+     * Suspends the current thread for a random number of millis between 0 and
+     * the indicated limit.
+     *
+     * @param ms max number of millis
+     */
+    public static void randomDelay(int ms) {
+        try {
+            Thread.sleep(random.nextInt(ms));
+        } catch (InterruptedException e) {
+            throw new RuntimeException("Interrupted", e);
+        }
+    }
+
+    /**
      * Suspends the current thread for a specified number of millis and nanos.
      * Suspends the current thread for a specified number of millis and nanos.
      *
      *
      * @param ms    number of millis
      * @param ms    number of millis