ONOS-1731 Fixed app gossip store & manager to deal with staggered app load on startup.

Change-Id: Ie88f5659646c11b906a63204f97083b6deb64adb

@@ -69,10 +69,16 @@ public class ApplicationManager implements ApplicationService, ApplicationAdminS
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected EventDeliveryService eventDispatcher;
     protected EventDeliveryService eventDispatcher;
 
 
+    private boolean initializing;
+
     @Activate
     @Activate
     public void activate() {
     public void activate() {
         eventDispatcher.addSink(ApplicationEvent.class, listenerRegistry);
         eventDispatcher.addSink(ApplicationEvent.class, listenerRegistry);
+
+        initializing = true;
         store.setDelegate(delegate);
         store.setDelegate(delegate);
+        initializing = false;
+
         log.info("Started");
         log.info("Started");
     }
     }
 
 
@@ -207,8 +213,12 @@ public class ApplicationManager implements ApplicationService, ApplicationAdminS
     private synchronized void installAppFeatures(Application app) throws Exception {
     private synchronized void installAppFeatures(Application app) throws Exception {
         for (String name : app.features()) {
         for (String name : app.features()) {
             Feature feature = featuresService.getFeature(name);
             Feature feature = featuresService.getFeature(name);
-            if (!featuresService.isInstalled(feature)) {
+            if (feature != null && !featuresService.isInstalled(feature)) {
                 featuresService.installFeature(name);
                 featuresService.installFeature(name);
+            } else if (feature == null && !initializing) {
+                // Suppress feature-not-found reporting during startup since these
+                // can arise naturally from the staggered cluster install.
+                log.warn("Feature {} not found", name);
             }
             }
         }
         }
     }
     }
@@ -216,8 +226,10 @@ public class ApplicationManager implements ApplicationService, ApplicationAdminS
     private synchronized void uninstallAppFeatures(Application app) throws Exception {
     private synchronized void uninstallAppFeatures(Application app) throws Exception {
         for (String name : app.features()) {
         for (String name : app.features()) {
             Feature feature = featuresService.getFeature(name);
             Feature feature = featuresService.getFeature(name);
-            if (featuresService.isInstalled(feature)) {
+            if (feature != null && featuresService.isInstalled(feature)) {
                 featuresService.uninstallFeature(name);
                 featuresService.uninstallFeature(name);
+            } else if (feature == null) {
+                log.warn("Feature {} not found", name);
             }
             }
         }
         }
     }
     }

@@ -85,6 +85,8 @@ import static org.slf4j.LoggerFactory.getLogger;
 public class GossipApplicationStore extends ApplicationArchive
 public class GossipApplicationStore extends ApplicationArchive
         implements ApplicationStore {
         implements ApplicationStore {
 
 
+    private static final int MAX_LOAD_RETRIES = 3;
+
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
 
 
     private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request");
     private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request");
@@ -163,11 +165,17 @@ public class GossipApplicationStore extends ApplicationArchive
      */
      */
     private void loadFromDisk() {
     private void loadFromDisk() {
         for (String name : getApplicationNames()) {
         for (String name : getApplicationNames()) {
+            for (int i = 0; i < MAX_LOAD_RETRIES; i++) {
+                try {
                     Application app = create(getApplicationDescription(name), false);
                     Application app = create(getApplicationDescription(name), false);
                     if (app != null && isActive(app.id().name())) {
                     if (app != null && isActive(app.id().name())) {
                         activate(app.id(), false);
                         activate(app.id(), false);
                         // load app permissions
                         // load app permissions
                     }
                     }
+                } catch (Exception e) {
+                    log.warn("Unable to load application {} from disk; retrying", name);
+                }
+            }
         }
         }
     }
     }