Added a temporary fix for out-of-order app activation event delivery.

Cleaning up app subsystem code a tiny bit as well.

Change-Id: I5df7d4c6d62d122653331474fb079648e779d595

@@ -80,8 +80,6 @@ public class ApplicationManager
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected FeaturesService featuresService;
     protected FeaturesService featuresService;
 
 
-    private boolean initializing;
-
     // Application supplied hooks for pre-activation processing.
     // Application supplied hooks for pre-activation processing.
     private final Multimap<String, Runnable> deactivateHooks = HashMultimap.create();
     private final Multimap<String, Runnable> deactivateHooks = HashMultimap.create();
     private final Cache<ApplicationId, CountDownLatch> pendingOperations =
     private final Cache<ApplicationId, CountDownLatch> pendingOperations =
@@ -92,11 +90,7 @@ public class ApplicationManager
     @Activate
     @Activate
     public void activate() {
     public void activate() {
         eventDispatcher.addSink(ApplicationEvent.class, listenerRegistry);
         eventDispatcher.addSink(ApplicationEvent.class, listenerRegistry);
-
-        initializing = true;
         store.setDelegate(delegate);
         store.setDelegate(delegate);
-        initializing = false;
-
         log.info("Started");
         log.info("Started");
     }
     }
 
 

@@ -30,6 +30,7 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.Tools;
 import org.onosproject.app.ApplicationDescription;
 import org.onosproject.app.ApplicationDescription;
 import org.onosproject.app.ApplicationEvent;
 import org.onosproject.app.ApplicationEvent;
 import org.onosproject.app.ApplicationException;
 import org.onosproject.app.ApplicationException;
@@ -89,11 +90,14 @@ import static org.slf4j.LoggerFactory.getLogger;
  * Manages inventory of applications in a distributed data store providing
  * Manages inventory of applications in a distributed data store providing
  * stronger consistency guarantees.
  * stronger consistency guarantees.
  */
  */
-@Component(immediate = true, enabled = true)
+@Component(immediate = true)
 @Service
 @Service
 public class DistributedApplicationStore extends ApplicationArchive
 public class DistributedApplicationStore extends ApplicationArchive
         implements ApplicationStore {
         implements ApplicationStore {
 
 
+    // FIXME: eliminate the need for this
+    private static final int FIXME_ACTIVATION_DELAY = 500;
+
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
 
 
     private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request");
     private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request");
@@ -144,7 +148,7 @@ public class DistributedApplicationStore extends ApplicationArchive
     public void activate() {
     public void activate() {
         messageHandlingExecutor = Executors.newSingleThreadExecutor(
         messageHandlingExecutor = Executors.newSingleThreadExecutor(
                 groupedThreads("onos/store/app", "message-handler", log));
                 groupedThreads("onos/store/app", "message-handler", log));
-        clusterCommunicator.<String, byte[]>addSubscriber(APP_BITS_REQUEST,
+        clusterCommunicator.addSubscriber(APP_BITS_REQUEST,
                                           bytes -> new String(bytes, Charsets.UTF_8),
                                           bytes -> new String(bytes, Charsets.UTF_8),
                                           name -> {
                                           name -> {
                                               try {
                                               try {
@@ -180,7 +184,7 @@ public class DistributedApplicationStore extends ApplicationArchive
      * Processes existing applications from the distributed map. This is done to
      * Processes existing applications from the distributed map. This is done to
      * account for events that this instance may be have missed due to a staggered start.
      * account for events that this instance may be have missed due to a staggered start.
      */
      */
-    void bootstrapExistingApplications() {
+    private void bootstrapExistingApplications() {
         apps.asJavaMap().forEach((appId, holder) -> setupApplicationAndNotify(appId, holder.app(), holder.state()));
         apps.asJavaMap().forEach((appId, holder) -> setupApplicationAndNotify(appId, holder.app(), holder.state()));
 
 
     }
     }
@@ -253,7 +257,7 @@ public class DistributedApplicationStore extends ApplicationArchive
     public void setDelegate(ApplicationStoreDelegate delegate) {
     public void setDelegate(ApplicationStoreDelegate delegate) {
         super.setDelegate(delegate);
         super.setDelegate(delegate);
         executor.execute(this::bootstrapExistingApplications);
         executor.execute(this::bootstrapExistingApplications);
-        executor.schedule(() -> loadFromDisk(), APP_LOAD_DELAY_MS, TimeUnit.MILLISECONDS);
+        executor.schedule((Runnable) this::loadFromDisk, APP_LOAD_DELAY_MS, TimeUnit.MILLISECONDS);
     }
     }
 
 
     @Override
     @Override
@@ -296,7 +300,7 @@ public class DistributedApplicationStore extends ApplicationArchive
     }
     }
 
 
     private boolean hasPrerequisites(ApplicationDescription app) {
     private boolean hasPrerequisites(ApplicationDescription app) {
-        return !app.requiredApps().stream().map(n -> getId(n))
+        return !app.requiredApps().stream().map(this::getId)
                 .anyMatch(id -> id == null || getApplication(id) == null);
                 .anyMatch(id -> id == null || getApplication(id) == null);
     }
     }
 
 
@@ -342,6 +346,11 @@ public class DistributedApplicationStore extends ApplicationArchive
             }
             }
             activateRequiredApps(vAppHolder.value().app());
             activateRequiredApps(vAppHolder.value().app());
 
 
+            // FIXME: Take a breath before the post-order operation to allow required app
+            // activation events to fully propagate. There appears to be an out-of-order
+            // event delivery issue that needs to be fixed.
+            Tools.delay(FIXME_ACTIVATION_DELAY);
+
             apps.computeIf(appId, v -> v != null && v.state() != ACTIVATED,
             apps.computeIf(appId, v -> v != null && v.state() != ACTIVATED,
                     (k, v) -> new InternalApplicationHolder(
                     (k, v) -> new InternalApplicationHolder(
                             v.app(), ACTIVATED, v.permissions()));
                             v.app(), ACTIVATED, v.permissions()));
@@ -557,7 +566,7 @@ public class DistributedApplicationStore extends ApplicationArchive
     /**
     /**
      * Internal class for holding app information.
      * Internal class for holding app information.
      */
      */
-    private static class InternalApplicationHolder {
+    private static final class InternalApplicationHolder {
         private final Application app;
         private final Application app;
         private final InternalState state;
         private final InternalState state;
         private final Set<Permission> permissions;
         private final Set<Permission> permissions;
@@ -569,7 +578,7 @@ public class DistributedApplicationStore extends ApplicationArchive
             permissions = null;
             permissions = null;
         }
         }
 
 
-        public InternalApplicationHolder(Application app, InternalState state, Set<Permission> permissions) {
+        private InternalApplicationHolder(Application app, InternalState state, Set<Permission> permissions) {
             this.app = Preconditions.checkNotNull(app);
             this.app = Preconditions.checkNotNull(app);
             this.state = state;
             this.state = state;
             this.permissions = permissions == null ? null : ImmutableSet.copyOf(permissions);
             this.permissions = permissions == null ? null : ImmutableSet.copyOf(permissions);