ONOS-785 Adding distributed store for apps & app admin CLIs

Change-Id: Ia7639f3258fca2a18ba513f0c95de0ab8ea7ceee

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cli.app;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.app.ApplicationAdminService;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.core.ApplicationId;
+
+/**
+ * Activates an installed application.
+ */
+@Command(scope = "onos", name = "app-activate",
+        description = "Activates an installed application")
+public class ApplicationActivateCommand extends AbstractShellCommand {
+
+    @Argument(index = 0, name = "name", description = "Application name",
+            required = true, multiValued = false)
+    String name = null;
+
+    @Override
+    protected void execute() {
+        ApplicationAdminService service = get(ApplicationAdminService.class);
+        ApplicationId appId = service.getId(name);
+        if (appId != null) {
+            service.activate(appId);
+        } else {
+            print("No such application: %s", name);
+        }
+    }
+
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cli.app;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.app.ApplicationAdminService;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.core.ApplicationId;
+
+/**
+ * Deactivates an installed application.
+ */
+@Command(scope = "onos", name = "app-deactivate",
+        description = "Deactivates an installed application")
+public class ApplicationDeactivateCommand extends AbstractShellCommand {
+
+    @Argument(index = 0, name = "name", description = "Application name",
+            required = true, multiValued = false)
+    String name = null;
+
+    @Override
+    protected void execute() {
+        ApplicationAdminService service = get(ApplicationAdminService.class);
+        ApplicationId appId = service.getId(name);
+        if (appId != null) {
+            service.deactivate(appId);
+        } else {
+            print("No such application: %s", name);
+        }
+    }
+
+}

@@ -13,12 +13,14 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.onosproject.cli;
+package org.onosproject.cli.app;
 
 import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fasterxml.jackson.databind.node.ArrayNode;
 import org.apache.karaf.shell.commands.Command;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.cli.Comparators;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.core.CoreService;
 
@@ -30,7 +32,7 @@ import static com.google.common.collect.Lists.newArrayList;
 /**
  * Lists application ID information.
  */
-@Command(scope = "onos", name = "apps",
+@Command(scope = "onos", name = "app-ids",
          description = "Lists application ID information")
 public class ApplicationIdListCommand extends AbstractShellCommand {
 

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cli.app;
+
+import org.apache.karaf.shell.console.Completer;
+import org.apache.karaf.shell.console.completer.StringsCompleter;
+import org.onosproject.app.ApplicationService;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.core.Application;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.SortedSet;
+
+/**
+ * Application name completer.
+ */
+public class ApplicationNameCompleter implements Completer {
+    @Override
+    public int complete(String buffer, int cursor, List<String> candidates) {
+        // Delegate string completer
+        StringsCompleter delegate = new StringsCompleter();
+
+        // Fetch our service and feed it's offerings to the string completer
+        ApplicationService service = AbstractShellCommand.get(ApplicationService.class);
+        Iterator<Application> it = service.getApplications().iterator();
+        SortedSet<String> strings = delegate.getStrings();
+        while (it.hasNext()) {
+            strings.add(it.next().id().name());
+        }
+
+        // Now let the completer do the work for figuring out what to offer.
+        return delegate.complete(buffer, cursor, candidates);
+    }
+
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.cli.app;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onosproject.app.ApplicationAdminService;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.core.ApplicationId;
+
+/**
+ * Uninstalls an application.
+ */
+@Command(scope = "onos", name = "app-uninstall",
+        description = "Uninstalls an application")
+public class ApplicationUninstallCommand extends AbstractShellCommand {
+
+    @Argument(index = 0, name = "name", description = "Application name",
+            required = true, multiValued = false)
+    String name = null;
+
+    @Override
+    protected void execute() {
+        ApplicationAdminService service = get(ApplicationAdminService.class);
+        ApplicationId appId = service.getId(name);
+        if (appId != null) {
+            service.uninstall(appId);
+        } else {
+            print("No such application: %s", name);
+        }
+    }
+
+}

@@ -13,32 +13,33 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package org.onosproject.cli;
+package org.onosproject.cli.app;
 
 import org.apache.karaf.shell.commands.Command;
+import org.onosproject.app.ApplicationService;
+import org.onosproject.cli.AbstractShellCommand;
+import org.onosproject.core.Application;
 
-import java.io.BufferedReader;
-import java.io.IOException;
-import java.io.InputStreamReader;
+import static org.onosproject.app.ApplicationState.ACTIVE;
 
 /**
- * Lists application ID information.
+ * Lists application information.
  */
-@Command(scope = "onos", name = "app-install",
-        description = "Lists application ID information")
-public class ApplicationInstallCommand extends AbstractShellCommand {
+@Command(scope = "onos", name = "apps",
+        description = "Lists application information")
+public class ApplicationsListCommand extends AbstractShellCommand {
+
+    private static final String FMT =
+            "%s id=%d, name=%s, version=%s, origin=%s, description=%s, " +
+                    "features=%s, featuresRepo=%s, permissions=%s";
 
     @Override
     protected void execute() {
-        // FIXME: merely an experiment for now
-        try (InputStreamReader isr = new InputStreamReader(System.in);
-             BufferedReader br = new BufferedReader(isr)) {
-            String line;
-            while ((line = br.readLine()) != null) {
-                print("%s", line.toUpperCase());
-            }
-        } catch (IOException e) {
-            e.printStackTrace();
+        ApplicationService service = get(ApplicationService.class);
+        for (Application app : service.getApplications()) {
+            print(FMT, service.getState(app.id()) == ACTIVE ? "*" : " ",
+                  app.id().id(), app.id().name(), app.version(), app.origin(),
+                  app.description(), app.features(), app.featuresRepo(), app.permissions());
         }
     }
 

@@ -21,7 +21,31 @@
         </command>
 
         <command>
-            <action class="org.onosproject.cli.ApplicationInstallCommand"/>
+            <action class="org.onosproject.cli.app.ApplicationsListCommand"/>
+        </command>
+
+        <command>
+            <action class="org.onosproject.cli.app.ApplicationActivateCommand"/>
+            <completers>
+                <ref component-id="appNameCompleter"/>
+                <null/>
+            </completers>
+        </command>
+
+        <command>
+            <action class="org.onosproject.cli.app.ApplicationDeactivateCommand"/>
+            <completers>
+                <ref component-id="appNameCompleter"/>
+                <null/>
+            </completers>
+        </command>
+
+        <command>
+            <action class="org.onosproject.cli.app.ApplicationUninstallCommand"/>
+            <completers>
+                <ref component-id="appNameCompleter"/>
+                <null/>
+            </completers>
         </command>
 
         <command>
@@ -77,7 +101,7 @@
             <action class="org.onosproject.cli.BalanceMastersCommand"/>
         </command>
         <command>
-            <action class="org.onosproject.cli.ApplicationIdListCommand"/>
+            <action class="org.onosproject.cli.app.ApplicationIdListCommand"/>
         </command>
 
         <command>
@@ -275,6 +299,7 @@
         </command>
     </command-bundle>
 
+    <bean id="appNameCompleter" class="org.onosproject.cli.app.ApplicationNameCompleter"/>
     <bean id="nodeIdCompleter" class="org.onosproject.cli.NodeIdCompleter"/>
     <bean id="deviceIdCompleter" class="org.onosproject.cli.net.DeviceIdCompleter"/>
     <bean id="clusterIdCompleter" class="org.onosproject.cli.net.ClusterIdCompleter"/>

@@ -39,6 +39,7 @@ import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
+import java.nio.file.NoSuchFileException;
 import java.util.Set;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
@@ -63,17 +64,25 @@ public class ApplicationArchive
 
     private static Logger log = LoggerFactory.getLogger(ApplicationArchive.class);
     private static final String APP_XML = "app.xml";
+    private static final String M2_PREFIX = "m2";
+
+    private static final String KARAF_ROOT = ".";
+    private static final String M2_ROOT = "system/";
     private static final String APPS_ROOT = "data/apps/";
 
-    private File appsDir = new File(APPS_ROOT);
+    private File karafRoot = new File(KARAF_ROOT);
+    private File m2Dir = new File(karafRoot, M2_ROOT);
+    private File appsDir = new File(karafRoot, APPS_ROOT);
 
     /**
      * Sets the root directory where application artifacts are kept.
      *
      * @param appsRoot top-level applications directory path
      */
-    protected void setAppsRoot(String appsRoot) {
-        this.appsDir = new File(appsRoot);
+    protected void setRootPath(String appsRoot) {
+        this.karafRoot = new File(appsRoot);
+        this.appsDir = new File(karafRoot, APPS_ROOT);
+        this.m2Dir = new File(karafRoot, M2_ROOT);
     }
 
     /**
@@ -81,8 +90,8 @@ public class ApplicationArchive
      *
      * @return top-level applications directory path
      */
-    protected String getAppsRoot() {
-        return appsDir.getPath();
+    protected String getRootPath() {
+        return karafRoot.getPath();
     }
 
     /**
@@ -238,10 +247,20 @@ public class ApplicationArchive
     }
 
     // Installs application artifacts into M2 repository.
-    private void installArtifacts(ApplicationDescription desc) {
-        // FIXME: implement M2 repository copy
+    private void installArtifacts(ApplicationDescription desc) throws IOException {
+        try {
+            Tools.copyDirectory(appFile(desc.name(), M2_PREFIX), m2Dir);
+        } catch (NoSuchFileException e) {
+            log.debug("Application {} has no M2 artifacts", desc.name());
+        }
     }
 
+    /**
+     * Marks the app as active by creating token file in the app directory.
+     *
+     * @param appName application name
+     * @return true if file was created
+     */
     protected boolean setActive(String appName) {
         try {
             return appFile(appName, "active").createNewFile();
@@ -250,10 +269,22 @@ public class ApplicationArchive
         }
     }
 
+    /**
+     * Clears the app as active by deleting token file in the app directory.
+     *
+     * @param appName application name
+     * @return true if file was deleted
+     */
     protected boolean clearActive(String appName) {
         return appFile(appName, "active").delete();
     }
 
+    /**
+     * Indicates whether the app was marked as active by checking for token file.
+     *
+     * @param appName application name
+     * @return true if the app is marked as active
+     */
     protected boolean isActive(String appName) {
         return appFile(appName, "active").exists();
     }

@@ -17,13 +17,17 @@ package org.onosproject.common.app;
 
 import com.google.common.collect.ImmutableSet;
 import com.google.common.io.ByteStreams;
+import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.onlab.util.Tools;
 import org.onosproject.app.ApplicationDescription;
 import org.onosproject.app.ApplicationException;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
+import java.util.Random;
 import java.util.Set;
 
 import static org.junit.Assert.assertArrayEquals;
@@ -32,13 +36,20 @@ import static org.onosproject.app.DefaultApplicationDescriptionTest.*;
 
 public class ApplicationArchiveTest {
 
-    static final String ROOT = "/tmp/app-junit";
+    static final String ROOT = "/tmp/app-junit/" + new Random().nextInt();
 
     private ApplicationArchive aar = new ApplicationArchive();
 
     @Before
     public void setUp() {
-        aar.setAppsRoot(ROOT);
+        aar.setRootPath(ROOT);
+    }
+
+    @After
+    public void tearDown() throws IOException {
+        if (new File(aar.getRootPath()).exists()) {
+            Tools.removeDirectory(aar.getRootPath());
+        }
     }
 
     private void validate(ApplicationDescription app) {
@@ -77,7 +88,8 @@ public class ApplicationArchiveTest {
     public void purgeApp() throws IOException {
         saveApp();
         aar.purgeApplication(APP_NAME);
-        assertEquals("incorrect names", ImmutableSet.of(), aar.getApplicationNames());
+        assertEquals("incorrect names", ImmutableSet.<String>of(),
+                     aar.getApplicationNames());
     }
 
     @Test

@@ -21,6 +21,7 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
+import org.apache.karaf.features.Feature;
 import org.apache.karaf.features.FeaturesService;
 import org.onosproject.app.ApplicationAdminService;
 import org.onosproject.app.ApplicationEvent;
@@ -202,14 +203,20 @@ public class ApplicationManager implements ApplicationService, ApplicationAdminS
 
     private void installAppFeatures(Application app) throws Exception {
         for (String name : app.features()) {
+            Feature feature = featuresService.getFeature(name);
+            if (!featuresService.isInstalled(feature)) {
                 featuresService.installFeature(name);
             }
         }
+    }
 
     private void uninstallAppFeatures(Application app) throws Exception {
         for (String name : app.features()) {
+            Feature feature = featuresService.getFeature(name);
+            if (featuresService.isInstalled(feature)) {
                 featuresService.uninstallFeature(name);
             }
         }
+    }
 
 }

@@ -43,6 +43,11 @@
             <artifactId>onlab-netty</artifactId>
             <version>${project.version}</version>
         </dependency>
+
+        <dependency>
+            <groupId>org.onosproject</groupId>
+            <artifactId>onos-core-common</artifactId>
+        </dependency>
 <!--
        <dependency>
             <groupId>net.kuujo.copycat</groupId>

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.store.app;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.util.KryoNamespace;
+import org.onosproject.app.ApplicationDescription;
+import org.onosproject.app.ApplicationEvent;
+import org.onosproject.app.ApplicationException;
+import org.onosproject.app.ApplicationState;
+import org.onosproject.app.ApplicationStore;
+import org.onosproject.cluster.ClusterService;
+import org.onosproject.cluster.ControllerNode;
+import org.onosproject.common.app.ApplicationArchive;
+import org.onosproject.core.Application;
+import org.onosproject.core.ApplicationId;
+import org.onosproject.core.ApplicationIdStore;
+import org.onosproject.core.DefaultApplication;
+import org.onosproject.core.Permission;
+import org.onosproject.store.cluster.messaging.ClusterCommunicationService;
+import org.onosproject.store.cluster.messaging.ClusterMessage;
+import org.onosproject.store.cluster.messaging.ClusterMessageHandler;
+import org.onosproject.store.cluster.messaging.MessageSubject;
+import org.onosproject.store.impl.EventuallyConsistentMap;
+import org.onosproject.store.impl.EventuallyConsistentMapEvent;
+import org.onosproject.store.impl.EventuallyConsistentMapImpl;
+import org.onosproject.store.impl.EventuallyConsistentMapListener;
+import org.onosproject.store.impl.WallclockClockManager;
+import org.onosproject.store.serializers.KryoNamespaces;
+import org.slf4j.Logger;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+
+import static com.google.common.io.ByteStreams.toByteArray;
+import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.onlab.util.Tools.namedThreads;
+import static org.onosproject.app.ApplicationEvent.Type.*;
+import static org.onosproject.store.app.GossipApplicationStore.InternalState.*;
+import static org.onosproject.store.impl.EventuallyConsistentMapEvent.Type.PUT;
+import static org.onosproject.store.impl.EventuallyConsistentMapEvent.Type.REMOVE;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Manages inventory of applications in a distributed data store that uses
+ * optimistic replication and gossip based anti-entropy techniques.
+ */
+@Component(immediate = true)
+@Service
+public class GossipApplicationStore extends ApplicationArchive
+        implements ApplicationStore {
+
+    private final Logger log = getLogger(getClass());
+
+    private static final MessageSubject APP_BITS_REQUEST = new MessageSubject("app-bits-request");
+
+    private static final int FETCH_TIMEOUT_MS = 10_000;
+    private static final int LOAD_TIMEOUT_MS = 5_000;
+
+    public enum InternalState {
+        INSTALLED, ACTIVATED, DEACTIVATED
+    }
+
+    private final ScheduledExecutorService executor =
+            Executors.newSingleThreadScheduledExecutor(namedThreads("onos-app-store"));
+
+    private EventuallyConsistentMap<ApplicationId, Application> apps;
+    private EventuallyConsistentMap<Application, InternalState> states;
+    private EventuallyConsistentMap<Application, Set<Permission>> permissions;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterCommunicationService clusterCommunicator;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterService clusterService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ApplicationIdStore idStore;
+
+    @Activate
+    public void activate() {
+        KryoNamespace.Builder intentSerializer = KryoNamespace.newBuilder()
+                .register(KryoNamespaces.API)
+                .register(InternalState.class);
+
+        clusterCommunicator.addSubscriber(APP_BITS_REQUEST, new InternalBitServer());
+
+        apps = new EventuallyConsistentMapImpl<>("apps", clusterService,
+                                                 clusterCommunicator,
+                                                 intentSerializer,
+                                                 new WallclockClockManager<>());
+
+        states = new EventuallyConsistentMapImpl<>("app-states",
+                                                   clusterService,
+                                                   clusterCommunicator,
+                                                   intentSerializer,
+                                                   new WallclockClockManager<>());
+        states.addListener(new InternalAppStatesListener());
+
+        permissions = new EventuallyConsistentMapImpl<>("app-permissions",
+                                                        clusterService,
+                                                        clusterCommunicator,
+                                                        intentSerializer,
+                                                        new WallclockClockManager<>());
+
+        // FIXME: figure out load from disk; this will require resolving the dual authority problem
+
+        executor.schedule(this::pruneUninstalledApps, LOAD_TIMEOUT_MS, MILLISECONDS);
+
+        log.info("Started");
+    }
+
+    private void loadFromDisk() {
+        for (String name : getApplicationNames()) {
+            create(getApplicationDescription(name));
+            // load app permissions
+        }
+    }
+
+    @Deactivate
+    public void deactivate() {
+        apps.destroy();
+        states.destroy();
+        permissions.destroy();
+        log.info("Stopped");
+    }
+
+    @Override
+    public Set<Application> getApplications() {
+        return ImmutableSet.copyOf(apps.values());
+    }
+
+    @Override
+    public ApplicationId getId(String name) {
+        return idStore.getAppId(name);
+    }
+
+    @Override
+    public Application getApplication(ApplicationId appId) {
+        return apps.get(appId);
+    }
+
+    @Override
+    public ApplicationState getState(ApplicationId appId) {
+        Application app = apps.get(appId);
+        InternalState s = app == null ? null : states.get(app);
+        return s == null ? null : s == ACTIVATED ?
+                ApplicationState.ACTIVE : ApplicationState.INSTALLED;
+    }
+
+    @Override
+    public Application create(InputStream appDescStream) {
+        ApplicationDescription appDesc = saveApplication(appDescStream);
+        return create(appDesc);
+    }
+
+    private Application create(ApplicationDescription appDesc) {
+        Application app = registerApp(appDesc);
+        apps.put(app.id(), app);
+        states.put(app, INSTALLED);
+        return app;
+    }
+
+    @Override
+    public void remove(ApplicationId appId) {
+        Application app = apps.get(appId);
+        if (app != null) {
+            apps.remove(appId);
+            states.remove(app);
+            permissions.remove(app);
+        }
+    }
+
+    @Override
+    public void activate(ApplicationId appId) {
+        Application app = apps.get(appId);
+        if (app != null) {
+            states.put(app, ACTIVATED);
+        }
+    }
+
+    @Override
+    public void deactivate(ApplicationId appId) {
+        Application app = apps.get(appId);
+        if (app != null) {
+            states.put(app, DEACTIVATED);
+        }
+    }
+
+    @Override
+    public Set<Permission> getPermissions(ApplicationId appId) {
+        Application app = apps.get(appId);
+        return app != null ? permissions.get(app) : null;
+    }
+
+    @Override
+    public void setPermissions(ApplicationId appId, Set<Permission> permissions) {
+        Application app = getApplication(appId);
+        if (app != null) {
+            this.permissions.put(app, permissions);
+            delegate.notify(new ApplicationEvent(APP_PERMISSIONS_CHANGED, app));
+        }
+    }
+
+    /**
+     * Listener to application state distributed map changes.
+     */
+    private final class InternalAppStatesListener
+            implements EventuallyConsistentMapListener<Application, InternalState> {
+        @Override
+        public void event(EventuallyConsistentMapEvent<Application, InternalState> event) {
+            Application app = event.key();
+            InternalState state = event.value();
+
+            if (event.type() == PUT) {
+                if (state == INSTALLED) {
+                    fetchBitsIfNeeded(app);
+                    delegate.notify(new ApplicationEvent(APP_INSTALLED, app));
+
+                } else if (state == ACTIVATED) {
+                    installAppIfNeeded(app);
+                    setActive(app.id().name());
+                    delegate.notify(new ApplicationEvent(APP_ACTIVATED, app));
+
+                } else if (state == DEACTIVATED) {
+                    clearActive(app.id().name());
+                    delegate.notify(new ApplicationEvent(APP_DEACTIVATED, app));
+                }
+            } else if (event.type() == REMOVE) {
+                delegate.notify(new ApplicationEvent(APP_UNINSTALLED, app));
+                purgeApplication(app.id().name());
+            }
+        }
+    }
+
+    /**
+     * Determines if the application bits are available locally.
+     */
+    private boolean appBitsAvailable(Application app) {
+        try {
+            ApplicationDescription appDesc = getApplicationDescription(app.id().name());
+            return appDesc.version().equals(app.version());
+        } catch (ApplicationException e) {
+            return false;
+        }
+    }
+
+    /**
+     * Fetches the bits from the cluster peers if necessary.
+     */
+    private void fetchBitsIfNeeded(Application app) {
+        if (!appBitsAvailable(app)) {
+            fetchBits(app);
+        }
+    }
+
+    /**
+     * Installs the application if necessary from the application peers.
+     */
+    private void installAppIfNeeded(Application app) {
+        if (!appBitsAvailable(app)) {
+            fetchBits(app);
+            delegate.notify(new ApplicationEvent(APP_INSTALLED, app));
+        }
+    }
+
+    /**
+     * Fetches the bits from the cluster peers.
+     */
+    private void fetchBits(Application app) {
+        ControllerNode localNode = clusterService.getLocalNode();
+        ClusterMessage message = new ClusterMessage(localNode.id(), APP_BITS_REQUEST,
+                                                    app.id().name().getBytes());
+        Map<ControllerNode, ListenableFuture<byte[]>> futures = new HashMap<>();
+        CountDownLatch latch = new CountDownLatch(1);
+
+        // FIXME: send message with name & version to make sure we don't get served old bits
+
+        log.info("Downloading bits for application {}", app.id().name());
+        for (ControllerNode node : clusterService.getNodes()) {
+            try {
+                ListenableFuture<byte[]> future = clusterCommunicator.sendAndReceive(message, node.id());
+                future.addListener(new InternalBitListener(app, node, future, latch), executor);
+            } catch (IOException e) {
+                log.debug("Unable to request bits for application {} from node {}",
+                          app.id().name(), node.id());
+            }
+        }
+
+        try {
+            if (!latch.await(FETCH_TIMEOUT_MS, MILLISECONDS)) {
+                log.warn("Unable to fetch bits for application {}", app.id().name());
+            }
+        } catch (InterruptedException e) {
+            log.warn("Interrupted while fetching bits for application {}", app.id().name());
+        }
+    }
+
+    /**
+     * Responder to requests for application bits.
+     */
+    private class InternalBitServer implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+            String name = new String(message.payload());
+            try {
+                message.respond(toByteArray(getApplicationInputStream(name)));
+            } catch (Exception e) {
+                log.debug("Unable to read bits for application {}", name);
+            }
+        }
+    }
+
+    /**
+     * Processes completed fetch requests.
+     */
+    private class InternalBitListener implements Runnable {
+        private final Application app;
+        private final ControllerNode node;
+        private final ListenableFuture<byte[]> future;
+        private final CountDownLatch latch;
+
+        public InternalBitListener(Application app, ControllerNode node,
+                                   ListenableFuture<byte[]> future, CountDownLatch latch) {
+            this.app = app;
+            this.node = node;
+            this.future = future;
+            this.latch = latch;
+        }
+
+        @Override
+        public void run() {
+            if (latch.getCount() > 0 && !future.isCancelled()) {
+                try {
+                    byte[] bits = future.get(1, MILLISECONDS);
+                    saveApplication(new ByteArrayInputStream(bits));
+                    log.info("Downloaded bits for application {} from node {}",
+                             app.id().name(), node.id());
+                    latch.countDown();
+                } catch (Exception e) {
+                    log.warn("Unable to fetch bits for application {} from node {}",
+                             app.id().name(), node.id());
+                }
+            }
+        }
+    }
+
+    /**
+     * Prunes applications which are not in the map, but are on disk.
+     */
+    private void pruneUninstalledApps() {
+        for (String name : getApplicationNames()) {
+            if (getApplication(getId(name)) == null) {
+                Application app = registerApp(getApplicationDescription(name));
+                delegate.notify(new ApplicationEvent(APP_UNINSTALLED, app));
+                purgeApplication(app.id().name());
+            }
+        }
+    }
+
+    /**
+     * Produces a registered application from the supplied description.
+     */
+    private Application registerApp(ApplicationDescription appDesc) {
+        ApplicationId appId = idStore.registerApplication(appDesc.name());
+        return new DefaultApplication(appId, appDesc.version(), appDesc.description(),
+                                      appDesc.origin(), appDesc.permissions(),
+                                      appDesc.featuresRepo(), appDesc.features());
+    }
+}
+

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Implementation of distributed applications store.
+ */
+package org.onosproject.store.app;

@@ -28,14 +28,17 @@ import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.onlab.util.KryoNamespace;
+import org.onosproject.app.ApplicationState;
 import org.onosproject.cluster.ControllerNode;
 import org.onosproject.cluster.DefaultControllerNode;
 import org.onosproject.cluster.Leadership;
 import org.onosproject.cluster.LeadershipEvent;
 import org.onosproject.cluster.NodeId;
 import org.onosproject.cluster.RoleInfo;
+import org.onosproject.core.DefaultApplication;
 import org.onosproject.core.DefaultApplicationId;
 import org.onosproject.core.DefaultGroupId;
+import org.onosproject.core.Version;
 import org.onosproject.mastership.MastershipTerm;
 import org.onosproject.net.ConnectPoint;
 import org.onosproject.net.DefaultAnnotations;
@@ -192,7 +195,10 @@ public final class KryoNamespaces {
             .register(MISC)
             .nextId(KryoNamespace.INITIAL_ID + 30 + 10)
             .register(
+                    Version.class,
                     ControllerNode.State.class,
+                    ApplicationState.class,
+                    DefaultApplication.class,
                     Device.Type.class,
                     Port.Type.class,
                     ChassisId.class,

@@ -8,4 +8,4 @@ export OCN="10.128.11.4"
 
 export OCI="${OC1}"
 
-export ONOS_FEATURES="webconsole,onos-api,onos-core,onos-cli,onos-openflow,onos-gui,onos-rest,onos-app-fwd,onos-app-proxyarp,onos-app-foo"
+export ONOS_FEATURES="webconsole,onos-api,onos-core,onos-cli,onos-openflow,onos-gui,onos-rest,onos-app-fwd,onos-app-proxyarp"

@@ -27,9 +27,11 @@ import java.io.InputStreamReader;
 import java.lang.Thread.UncaughtExceptionHandler;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.SimpleFileVisitor;
+import java.nio.file.StandardCopyOption;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.util.ArrayList;
 import java.util.List;
@@ -200,4 +202,60 @@ public abstract class Tools {
         }
     }
 
+
+    /**
+     * Copies the specified directory path. Use with great caution since
+     * no attempt is made to check for symbolic links, which could result in
+     * copy of unintended files.
+     *
+     * @param src directory to be copied
+     * @param dst destination directory to be removed
+     * @throws java.io.IOException if unable to remove contents
+     */
+    public static void copyDirectory(String src, String dst) throws IOException {
+        walkFileTree(Paths.get(src), new DirectoryCopier(src, dst));
+    }
+
+    /**
+     * Copies the specified directory path. Use with great caution since
+     * no attempt is made to check for symbolic links, which could result in
+     * copy of unintended files.
+     *
+     * @param src directory to be copied
+     * @param dst destination directory to be removed
+     * @throws java.io.IOException if unable to remove contents
+     */
+    public static void copyDirectory(File src, File dst) throws IOException {
+        walkFileTree(Paths.get(src.getAbsolutePath()),
+                     new DirectoryCopier(src.getAbsolutePath(),
+                                         dst.getAbsolutePath()));
+    }
+
+
+    public static class DirectoryCopier extends SimpleFileVisitor<Path> {
+        private Path src;
+        private Path dst;
+        private StandardCopyOption copyOption = StandardCopyOption.REPLACE_EXISTING;
+
+        DirectoryCopier(String src, String dst) {
+            this.src = Paths.get(src);
+            this.dst = Paths.get(dst);
+        }
+
+        @Override
+        public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
+            Path targetPath = dst.resolve(src.relativize(dir));
+            if (!Files.exists(targetPath)) {
+                Files.createDirectory(targetPath);
+            }
+            return FileVisitResult.CONTINUE;
+        }
+
+        @Override
+        public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+            Files.copy(file, dst.resolve(src.relativize(file)), copyOption);
+            return FileVisitResult.CONTINUE;
+        }
+    }
+
 }