Committed by
Gerrit Code Review
Increased number of retries and added variable delay to better withstand StorageException timeout.
Change-Id: Ic33d4e66e2787c717f7e3c90879d725a6e3e74d7
Showing
3 changed files
with
23 additions
and
7 deletions
... | @@ -62,8 +62,8 @@ import java.util.function.Function; | ... | @@ -62,8 +62,8 @@ import java.util.function.Function; |
62 | 62 | ||
63 | import static com.google.common.io.ByteStreams.toByteArray; | 63 | import static com.google.common.io.ByteStreams.toByteArray; |
64 | import static java.util.concurrent.TimeUnit.MILLISECONDS; | 64 | import static java.util.concurrent.TimeUnit.MILLISECONDS; |
65 | -import static org.onlab.util.Tools.delay; | ||
66 | import static org.onlab.util.Tools.groupedThreads; | 65 | import static org.onlab.util.Tools.groupedThreads; |
66 | +import static org.onlab.util.Tools.randomDelay; | ||
67 | import static org.onosproject.app.ApplicationEvent.Type.*; | 67 | import static org.onosproject.app.ApplicationEvent.Type.*; |
68 | import static org.onosproject.store.app.GossipApplicationStore.InternalState.*; | 68 | import static org.onosproject.store.app.GossipApplicationStore.InternalState.*; |
69 | import static org.onosproject.store.service.EventuallyConsistentMapEvent.Type.PUT; | 69 | import static org.onosproject.store.service.EventuallyConsistentMapEvent.Type.PUT; |
... | @@ -83,11 +83,10 @@ public class GossipApplicationStore extends ApplicationArchive | ... | @@ -83,11 +83,10 @@ public class GossipApplicationStore extends ApplicationArchive |
83 | 83 | ||
84 | private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request"); | 84 | private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request"); |
85 | 85 | ||
86 | - private static final int MAX_LOAD_RETRIES = 3; | 86 | + private static final int MAX_LOAD_RETRIES = 5; |
87 | private static final int RETRY_DELAY_MS = 2_000; | 87 | private static final int RETRY_DELAY_MS = 2_000; |
88 | 88 | ||
89 | private static final int FETCH_TIMEOUT_MS = 10_000; | 89 | private static final int FETCH_TIMEOUT_MS = 10_000; |
90 | - private static final int LOAD_TIMEOUT_MS = 5_000; | ||
91 | 90 | ||
92 | public enum InternalState { | 91 | public enum InternalState { |
93 | INSTALLED, ACTIVATED, DEACTIVATED | 92 | INSTALLED, ACTIVATED, DEACTIVATED |
... | @@ -169,7 +168,7 @@ public class GossipApplicationStore extends ApplicationArchive | ... | @@ -169,7 +168,7 @@ public class GossipApplicationStore extends ApplicationArchive |
169 | } | 168 | } |
170 | } catch (Exception e) { | 169 | } catch (Exception e) { |
171 | log.warn("Unable to load application {} from disk; retrying", name); | 170 | log.warn("Unable to load application {} from disk; retrying", name); |
172 | - delay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake | 171 | + randomDelay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake |
173 | } | 172 | } |
174 | } | 173 | } |
175 | } | 174 | } | ... | ... |
... | @@ -16,7 +16,7 @@ import org.slf4j.Logger; | ... | @@ -16,7 +16,7 @@ import org.slf4j.Logger; |
16 | 16 | ||
17 | import java.util.Map; | 17 | import java.util.Map; |
18 | 18 | ||
19 | -import static org.onlab.util.Tools.delay; | 19 | +import static org.onlab.util.Tools.randomDelay; |
20 | import static org.slf4j.LoggerFactory.getLogger; | 20 | import static org.slf4j.LoggerFactory.getLogger; |
21 | 21 | ||
22 | /** | 22 | /** |
... | @@ -26,7 +26,7 @@ import static org.slf4j.LoggerFactory.getLogger; | ... | @@ -26,7 +26,7 @@ import static org.slf4j.LoggerFactory.getLogger; |
26 | @Service | 26 | @Service |
27 | public class ConsistentIdBlockStore implements IdBlockStore { | 27 | public class ConsistentIdBlockStore implements IdBlockStore { |
28 | 28 | ||
29 | - private static final int MAX_TRIES = 3; | 29 | + private static final int MAX_TRIES = 5; |
30 | private static final int RETRY_DELAY_MS = 2_000; | 30 | private static final int RETRY_DELAY_MS = 2_000; |
31 | 31 | ||
32 | private final Logger log = getLogger(getClass()); | 32 | private final Logger log = getLogger(getClass()); |
... | @@ -63,7 +63,7 @@ public class ConsistentIdBlockStore implements IdBlockStore { | ... | @@ -63,7 +63,7 @@ public class ConsistentIdBlockStore implements IdBlockStore { |
63 | log.warn("Unable to allocate ID block due to {}; retrying...", | 63 | log.warn("Unable to allocate ID block due to {}; retrying...", |
64 | e.getMessage()); | 64 | e.getMessage()); |
65 | exc = e; | 65 | exc = e; |
66 | - delay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake | 66 | + randomDelay(RETRY_DELAY_MS); // FIXME: This is a deliberate hack; fix in Drake |
67 | } | 67 | } |
68 | } | 68 | } |
69 | throw new IllegalStateException("Unable to allocate ID block", exc); | 69 | throw new IllegalStateException("Unable to allocate ID block", exc); | ... | ... |
... | @@ -39,6 +39,7 @@ import java.util.Arrays; | ... | @@ -39,6 +39,7 @@ import java.util.Arrays; |
39 | import java.util.Collection; | 39 | import java.util.Collection; |
40 | import java.util.Dictionary; | 40 | import java.util.Dictionary; |
41 | import java.util.List; | 41 | import java.util.List; |
42 | +import java.util.Random; | ||
42 | import java.util.concurrent.CompletableFuture; | 43 | import java.util.concurrent.CompletableFuture; |
43 | import java.util.concurrent.ExecutionException; | 44 | import java.util.concurrent.ExecutionException; |
44 | import java.util.concurrent.Future; | 45 | import java.util.concurrent.Future; |
... | @@ -62,6 +63,8 @@ public abstract class Tools { | ... | @@ -62,6 +63,8 @@ public abstract class Tools { |
62 | 63 | ||
63 | private static final Logger log = getLogger(Tools.class); | 64 | private static final Logger log = getLogger(Tools.class); |
64 | 65 | ||
66 | + private static Random random = new Random(); | ||
67 | + | ||
65 | /** | 68 | /** |
66 | * Returns a thread factory that produces threads named according to the | 69 | * Returns a thread factory that produces threads named according to the |
67 | * supplied name pattern. | 70 | * supplied name pattern. |
... | @@ -195,6 +198,20 @@ public abstract class Tools { | ... | @@ -195,6 +198,20 @@ public abstract class Tools { |
195 | } | 198 | } |
196 | 199 | ||
197 | /** | 200 | /** |
201 | + * Suspends the current thread for a random number of millis between 0 and | ||
202 | + * the indicated limit. | ||
203 | + * | ||
204 | + * @param ms max number of millis | ||
205 | + */ | ||
206 | + public static void randomDelay(int ms) { | ||
207 | + try { | ||
208 | + Thread.sleep(random.nextInt(ms)); | ||
209 | + } catch (InterruptedException e) { | ||
210 | + throw new RuntimeException("Interrupted", e); | ||
211 | + } | ||
212 | + } | ||
213 | + | ||
214 | + /** | ||
198 | * Suspends the current thread for a specified number of millis and nanos. | 215 | * Suspends the current thread for a specified number of millis and nanos. |
199 | * | 216 | * |
200 | * @param ms number of millis | 217 | * @param ms number of millis | ... | ... |
-
Please register or login to post a comment