Merge branch 'master' of ssh://gerrit.onlab.us:29418/onos-next

@@ -18,18 +18,20 @@
 
 
     <dependencies>
     <dependencies>
       <dependency>
       <dependency>
-	<groupId>org.codehaus.jackson</groupId>
-	<artifactId>jackson-core-asl</artifactId>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-core-asl</artifactId>
       </dependency>
       </dependency>
       <dependency>
       <dependency>
-	<groupId>org.codehaus.jackson</groupId>
-	<artifactId>jackson-mapper-asl</artifactId>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-mapper-asl</artifactId>
       </dependency>
       </dependency>
       <dependency>
       <dependency>
         <groupId>com.fasterxml.jackson.core</groupId>
         <groupId>com.fasterxml.jackson.core</groupId>
         <artifactId>jackson-annotations</artifactId>
         <artifactId>jackson-annotations</artifactId>
-        <version>2.4.2</version>
-        <scope>provided</scope>
+      </dependency>
+      <dependency>
+        <groupId>org.onlab.onos</groupId>
+        <artifactId>onlab-misc</artifactId>
       </dependency>
       </dependency>
     </dependencies> 
     </dependencies> 
 
 

@@ -41,5 +41,17 @@
             <groupId>org.apache.karaf.shell</groupId>
             <groupId>org.apache.karaf.shell</groupId>
             <artifactId>org.apache.karaf.shell.console</artifactId>
             <artifactId>org.apache.karaf.shell.console</artifactId>
         </dependency>
         </dependency>
+        <dependency>
+            <groupId>org.onlab.onos</groupId>
+            <artifactId>onlab-misc</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.osgi</groupId>
+            <artifactId>org.osgi.core</artifactId>
+        </dependency>
     </dependencies>
     </dependencies>
 </project>
 </project>

@@ -4,8 +4,6 @@ import java.io.IOException;
 
 
 import org.onlab.netty.Message;
 import org.onlab.netty.Message;
 import org.onlab.netty.MessageHandler;
 import org.onlab.netty.MessageHandler;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 
 
 
 /**
 /**
@@ -13,11 +11,8 @@ import org.slf4j.LoggerFactory;
  */
  */
 public class NettyEchoHandler implements MessageHandler {
 public class NettyEchoHandler implements MessageHandler {
 
 
-    private final Logger log = LoggerFactory.getLogger(getClass());
-
     @Override
     @Override
     public void handle(Message message) throws IOException {
     public void handle(Message message) throws IOException {
-        //log.info("Received message. Echoing it back to the sender.");
         message.respond(message.payload());
         message.respond(message.payload());
     }
     }
 }
 }

@@ -8,12 +8,12 @@ import org.slf4j.LoggerFactory;
 /**
 /**
  * A MessageHandler that simply logs the information.
  * A MessageHandler that simply logs the information.
  */
  */
-public class NettyLoggingHandler implements MessageHandler {
+public class NettyNothingHandler implements MessageHandler {
 
 
     private final Logger log = LoggerFactory.getLogger(getClass());
     private final Logger log = LoggerFactory.getLogger(getClass());
 
 
     @Override
     @Override
     public void handle(Message message) {
     public void handle(Message message) {
-        //log.info("Received message. Payload has {} bytes", message.payload().length);
+      // Do nothing
     }
     }
 }
 }

 package org.onlab.onos.foo;
 package org.onlab.onos.foo;
 
 
+import static java.lang.Thread.sleep;
+
 import java.io.IOException;
 import java.io.IOException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.TimeoutException;
 
 
 import org.onlab.metrics.MetricsComponent;
 import org.onlab.metrics.MetricsComponent;
@@ -15,14 +18,29 @@ import org.slf4j.LoggerFactory;
 
 
 import com.codahale.metrics.Timer;
 import com.codahale.metrics.Timer;
 
 
+/**
+ * The Simple netty client test.
+ */
 // FIXME: Should be move out to test or app
 // FIXME: Should be move out to test or app
 public final class SimpleNettyClient {
 public final class SimpleNettyClient {
 
 
 private static Logger log = LoggerFactory.getLogger(SimpleNettyClient.class);
 private static Logger log = LoggerFactory.getLogger(SimpleNettyClient.class);
 
 
+    static NettyMessagingService messaging;
+    static MetricsManager metrics;
+
         private SimpleNettyClient() {
         private SimpleNettyClient() {
     }
     }
 
 
+    /**
+     * The entry point of application.
+     *
+     * @param args the input arguments
+     * @throws IOException the iO exception
+     * @throws InterruptedException the interrupted exception
+     * @throws ExecutionException the execution exception
+     * @throws TimeoutException the timeout exception
+     */
     public static void main(String[] args)
     public static void main(String[] args)
             throws IOException, InterruptedException, ExecutionException,
             throws IOException, InterruptedException, ExecutionException,
             TimeoutException {
             TimeoutException {
@@ -34,48 +52,87 @@ private static Logger log = LoggerFactory.getLogger(SimpleNettyClient.class);
 
 
         System.exit(0);
         System.exit(0);
     }
     }
-    public static void startStandalone(String... args) throws Exception {
+
+    /**
+     * Start standalone.
+     *
+     * @param args the args
+     * @throws Exception the exception
+     */
+    public static void startStandalone(String[] args) throws Exception {
         String host = args.length > 0 ? args[0] : "localhost";
         String host = args.length > 0 ? args[0] : "localhost";
         int port = args.length > 1 ? Integer.parseInt(args[1]) : 8081;
         int port = args.length > 1 ? Integer.parseInt(args[1]) : 8081;
         int warmup = args.length > 2 ? Integer.parseInt(args[2]) : 1000;
         int warmup = args.length > 2 ? Integer.parseInt(args[2]) : 1000;
         int iterations = args.length > 3 ? Integer.parseInt(args[3]) : 50 * 100000;
         int iterations = args.length > 3 ? Integer.parseInt(args[3]) : 50 * 100000;
-        NettyMessagingService messaging = new TestNettyMessagingService(9081);
-        MetricsManager metrics = new MetricsManager();
+        messaging = new TestNettyMessagingService(9081);
+        metrics = new MetricsManager();
         Endpoint endpoint = new Endpoint(host, port);
         Endpoint endpoint = new Endpoint(host, port);
         messaging.activate();
         messaging.activate();
         metrics.activate();
         metrics.activate();
         MetricsFeature feature = new MetricsFeature("latency");
         MetricsFeature feature = new MetricsFeature("latency");
         MetricsComponent component = metrics.registerComponent("NettyMessaging");
         MetricsComponent component = metrics.registerComponent("NettyMessaging");
-        log.info("warmup....");
+        log.info("connecting " + host + ":" + port + " warmup:" + warmup + " iterations:" + iterations);
 
 
         for (int i = 0; i < warmup; i++) {
         for (int i = 0; i < warmup; i++) {
             messaging.sendAsync(endpoint, "simple", "Hello World".getBytes());
             messaging.sendAsync(endpoint, "simple", "Hello World".getBytes());
             Response response = messaging
             Response response = messaging
                     .sendAndReceive(endpoint, "echo",
                     .sendAndReceive(endpoint, "echo",
                             "Hello World".getBytes());
                             "Hello World".getBytes());
+            response.get(100000, TimeUnit.MILLISECONDS);
+        }
+
+        log.info("measuring round-trip send & receive");
+        Timer sendAndReceiveTimer = metrics.createTimer(component, feature, "SendAndReceive");
+        int timeouts = 0;
+
+        for (int i = 0; i < iterations; i++) {
+            Response response;
+            Timer.Context context = sendAndReceiveTimer.time();
+            try {
+                response = messaging
+                        .sendAndReceive(endpoint, "echo",
+                                "Hello World".getBytes());
+                response.get(10000, TimeUnit.MILLISECONDS);
+            } catch (TimeoutException e) {
+                timeouts++;
+                log.info("timeout:" + timeouts + " at iteration:" + i);
+            } finally {
+                context.stop();
+            }
+            // System.out.println("Got back:" + new String(response.get(2, TimeUnit.SECONDS)));
         }
         }
 
 
+        //sleep(1000);
         log.info("measuring async sender");
         log.info("measuring async sender");
         Timer sendAsyncTimer = metrics.createTimer(component, feature, "AsyncSender");
         Timer sendAsyncTimer = metrics.createTimer(component, feature, "AsyncSender");
 
 
         for (int i = 0; i < iterations; i++) {
         for (int i = 0; i < iterations; i++) {
-            Timer.Context context = sendAsyncTimer.time();
-            messaging.sendAsync(endpoint, "simple", "Hello World".getBytes());
-            context.stop();
+        Timer.Context context = sendAsyncTimer.time();
+        messaging.sendAsync(endpoint, "simple", "Hello World".getBytes());
+        context.stop();
         }
         }
+        sleep(10000);
+    }
 
 
-        Timer sendAndReceiveTimer = metrics.createTimer(component, feature, "SendAndReceive");
-        for (int i = 0; i < iterations; i++) {
-            Timer.Context context = sendAndReceiveTimer.time();
-            Response response = messaging
-                    .sendAndReceive(endpoint, "echo",
-                                    "Hello World".getBytes());
-            // System.out.println("Got back:" + new String(response.get(2, TimeUnit.SECONDS)));
-            context.stop();
+    public static void stop() {
+        try {
+            messaging.deactivate();
+            metrics.deactivate();
+        } catch (Exception e) {
+            log.info("Unable to stop client %s", e);
         }
         }
     }
     }
 
 
+    /**
+     * The type Test netty messaging service.
+     */
     public static class TestNettyMessagingService extends NettyMessagingService {
     public static class TestNettyMessagingService extends NettyMessagingService {
+        /**
+         * Instantiates a new Test netty messaging service.
+         *
+         * @param port the port
+         * @throws Exception the exception
+         */
         public TestNettyMessagingService(int port) throws Exception {
         public TestNettyMessagingService(int port) throws Exception {
             super(port);
             super(port);
         }
         }

 package org.onlab.onos.foo;
 package org.onlab.onos.foo;
 
 
 import static org.onlab.onos.foo.SimpleNettyClient.startStandalone;
 import static org.onlab.onos.foo.SimpleNettyClient.startStandalone;
+import static org.onlab.onos.foo.SimpleNettyClient.stop;
 
 
 import org.apache.karaf.shell.commands.Argument;
 import org.apache.karaf.shell.commands.Argument;
 import org.apache.karaf.shell.commands.Command;
 import org.apache.karaf.shell.commands.Command;
@@ -10,7 +11,7 @@ import org.onlab.onos.cli.AbstractShellCommand;
  * Test Netty client performance.
  * Test Netty client performance.
  */
  */
 @Command(scope = "onos", name = "simple-netty-client",
 @Command(scope = "onos", name = "simple-netty-client",
-        description = "Starts the simple Netty client")
+        description = "Starts simple Netty client")
 public class SimpleNettyClientCommand extends AbstractShellCommand {
 public class SimpleNettyClientCommand extends AbstractShellCommand {
 
 
     //FIXME: replace these arguments with proper ones needed for the test.
     //FIXME: replace these arguments with proper ones needed for the test.
@@ -18,17 +19,17 @@ public class SimpleNettyClientCommand extends AbstractShellCommand {
             required = false, multiValued = false)
             required = false, multiValued = false)
     String hostname = "localhost";
     String hostname = "localhost";
 
 
-    @Argument(index = 3, name = "port", description = "Port",
+    @Argument(index = 1, name = "port", description = "Port",
             required = false, multiValued = false)
             required = false, multiValued = false)
     String port = "8081";
     String port = "8081";
 
 
-    @Argument(index = 1, name = "warmupCount", description = "Warm-up count",
+    @Argument(index = 2, name = "warmupCount", description = "Warm-up count",
             required = false, multiValued = false)
             required = false, multiValued = false)
     String warmupCount = "1000";
     String warmupCount = "1000";
 
 
-    @Argument(index = 2, name = "messageCount", description = "Message count",
+    @Argument(index = 3, name = "messageCount", description = "Message count",
             required = false, multiValued = false)
             required = false, multiValued = false)
-    String messageCount = "100000";
+    String messageCount = "1000000";
 
 
     @Override
     @Override
     protected void execute() {
     protected void execute() {
@@ -37,5 +38,6 @@ public class SimpleNettyClientCommand extends AbstractShellCommand {
         } catch (Exception e) {
         } catch (Exception e) {
             error("Unable to start client %s", e);
             error("Unable to start client %s", e);
         }
         }
+        stop();
     }
     }
 }
 }

@@ -12,16 +12,30 @@ import org.slf4j.LoggerFactory;
 
 
             private SimpleNettyServer() {}
             private SimpleNettyServer() {}
 
 
-            public static void main(String... args) throws Exception {
+    /**
+     * The entry point of application.
+     *
+     * @param args the input arguments
+     * @throws Exception the exception
+     */
+    public static void main(String... args) throws Exception {
                 startStandalone(args);
                 startStandalone(args);
                 System.exit(0);
                 System.exit(0);
             }
             }
 
 
-        public static void startStandalone(String[] args) throws Exception {
-            NettyMessagingService server = new NettyMessagingService(8081);
+    /**
+     * Start standalone server.
+     *
+     * @param args the args
+     * @throws Exception the exception
+     */
+    public static void startStandalone(String[] args) throws Exception {
+            int port = args.length > 0 ? Integer.parseInt(args[0]) : 8081;
+            NettyMessagingService server = new NettyMessagingService(port);
             server.activate();
             server.activate();
-            server.registerHandler("simple", new NettyLoggingHandler());
+            server.registerHandler("simple", new NettyNothingHandler());
             server.registerHandler("echo", new NettyEchoHandler());
             server.registerHandler("echo", new NettyEchoHandler());
+            log.info("Netty Server server on port " + port);
         }
         }
     }
     }
 
 

@@ -10,26 +10,18 @@ import org.onlab.onos.cli.AbstractShellCommand;
  * Starts the Simple Netty server.
  * Starts the Simple Netty server.
  */
  */
 @Command(scope = "onos", name = "simple-netty-server",
 @Command(scope = "onos", name = "simple-netty-server",
-         description = "Starts the simple netty server")
+         description = "Starts simple Netty server")
 public class SimpleNettyServerCommand extends AbstractShellCommand {
 public class SimpleNettyServerCommand extends AbstractShellCommand {
 
 
     //FIXME: Replace these with parameters for
     //FIXME: Replace these with parameters for
-    @Argument(index = 0, name = "serverIp", description = "Server IP address",
+    @Argument(index = 0, name = "port", description = "listen port",
               required = false, multiValued = false)
               required = false, multiValued = false)
-    String serverIp = "127.0.0.1";
-
-    @Argument(index = 1, name = "workers", description = "IO workers",
-              required = false, multiValued = false)
-    String workers = "6";
-
-    @Argument(index = 2, name = "messageLength", description = "Message length (bytes)",
-              required = false, multiValued = false)
-    String messageLength = "128";
+    String port = "8081";
 
 
     @Override
     @Override
     protected void execute() {
     protected void execute() {
         try {
         try {
-            startStandalone(new String[]{serverIp, workers, messageLength});
+            startStandalone(new String[]{port});
         } catch (Exception e) {
         } catch (Exception e) {
             error("Unable to start server %s", e);
             error("Unable to start server %s", e);
         }
         }

@@ -10,6 +10,7 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.ApplicationId;
+import org.onlab.onos.CoreService;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.HostId;
 import org.onlab.onos.net.HostId;
 import org.onlab.onos.net.Path;
 import org.onlab.onos.net.Path;
@@ -53,13 +54,16 @@ public class ReactiveForwarding {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected FlowRuleService flowRuleService;
     protected FlowRuleService flowRuleService;
 
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
+
     private ReactivePacketProcessor processor = new ReactivePacketProcessor();
     private ReactivePacketProcessor processor = new ReactivePacketProcessor();
 
 
     private ApplicationId appId;
     private ApplicationId appId;
 
 
     @Activate
     @Activate
     public void activate() {
     public void activate() {
-        appId = ApplicationId.getAppId();
+        appId = coreService.registerApplication("org.onlab.onos.fwd");
         packetService.addProcessor(processor, PacketProcessor.ADVISOR_MAX + 2);
         packetService.addProcessor(processor, PacketProcessor.ADVISOR_MAX + 2);
         log.info("Started with Application ID {}", appId.id());
         log.info("Started with Application ID {}", appId.id());
     }
     }
@@ -166,8 +170,6 @@ public class ReactiveForwarding {
         // We don't yet support bufferids in the flowservice so packet out first.
         // We don't yet support bufferids in the flowservice so packet out first.
         packetOut(context, portNumber);
         packetOut(context, portNumber);
 
 
-
-
         // Install the flow rule to handle this type of message from now on.
         // Install the flow rule to handle this type of message from now on.
         Ethernet inPkt = context.inPacket().parsed();
         Ethernet inPkt = context.inPacket().parsed();
         TrafficSelector.Builder builder = DefaultTrafficSelector.builder();
         TrafficSelector.Builder builder = DefaultTrafficSelector.builder();

@@ -16,4 +16,14 @@
 
 
     <description>ONOS simple Mobility app</description>
     <description>ONOS simple Mobility app</description>
 
 
+    <dependencies>
+      <dependency>
+        <groupId>com.google.guava</groupId>
+        <artifactId>guava</artifactId>
+      </dependency>
+      <dependency>
+        <groupId>org.onlab.onos</groupId>
+        <artifactId>onlab-misc</artifactId>
+      </dependency>
+    </dependencies>
 </project>
 </project>

@@ -10,6 +10,7 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.ApplicationId;
+import org.onlab.onos.CoreService;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.device.DeviceService;
 import org.onlab.onos.net.device.DeviceService;
@@ -44,11 +45,14 @@ public class HostMobility {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected DeviceService deviceService;
     protected DeviceService deviceService;
 
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
+
     private ApplicationId appId;
     private ApplicationId appId;
 
 
     @Activate
     @Activate
     public void activate() {
     public void activate() {
-        appId = ApplicationId.getAppId();
+        appId = coreService.registerApplication("org.onlab.onos.mobility");
         hostService.addListener(new InternalHostListener());
         hostService.addListener(new InternalHostListener());
         log.info("Started with Application ID {}", appId.id());
         log.info("Started with Application ID {}", appId.id());
     }
     }

@@ -23,7 +23,8 @@
         <module>foo</module>
         <module>foo</module>
         <module>mobility</module>
         <module>mobility</module>
         <module>proxyarp</module>
         <module>proxyarp</module>
-	    <module>config</module>
+        <module>config</module>
+        <module>sdnip</module>
     </modules>
     </modules>
 
 
     <properties>
     <properties>

@@ -8,6 +8,7 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.ApplicationId;
+import org.onlab.onos.CoreService;
 import org.onlab.onos.net.packet.PacketContext;
 import org.onlab.onos.net.packet.PacketContext;
 import org.onlab.onos.net.packet.PacketProcessor;
 import org.onlab.onos.net.packet.PacketProcessor;
 import org.onlab.onos.net.packet.PacketService;
 import org.onlab.onos.net.packet.PacketService;
@@ -31,11 +32,14 @@ public class ProxyArp {
 
 
     private ProxyArpProcessor processor = new ProxyArpProcessor();
     private ProxyArpProcessor processor = new ProxyArpProcessor();
 
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
+
     private ApplicationId appId;
     private ApplicationId appId;
 
 
     @Activate
     @Activate
     public void activate() {
     public void activate() {
-        appId = ApplicationId.getAppId();
+        appId = coreService.registerApplication("org.onlab.onos.proxyarp");
         packetService.addProcessor(processor, PacketProcessor.ADVISOR_MAX + 1);
         packetService.addProcessor(processor, PacketProcessor.ADVISOR_MAX + 1);
         log.info("Started with Application ID {}", appId.id());
         log.info("Started with Application ID {}", appId.id());
     }
     }

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.onlab.onos</groupId>
+        <artifactId>onos-apps</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>onos-app-sdnip</artifactId>
+    <packaging>bundle</packaging>
+
+    <description>SDN-IP peering application</description>
+
+    <dependencies>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-core-asl</artifactId>
+      </dependency>
+      <dependency>
+        <groupId>org.codehaus.jackson</groupId>
+        <artifactId>jackson-mapper-asl</artifactId>
+      </dependency>
+      <dependency>
+        <groupId>com.fasterxml.jackson.core</groupId>
+        <artifactId>jackson-annotations</artifactId>
+        <version>2.4.2</version>
+        <scope>provided</scope>
+      </dependency>
+    </dependencies> 
+
+</project>

+package org.onlab.onos.sdnip;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.slf4j.Logger;
+
+/**
+ * Placeholder SDN-IP component.
+ */
+@Component(immediate = true)
+public class SdnIp {
+
+    private final Logger log = getLogger(getClass());
+
+    @Activate
+    protected void activate() {
+        log.debug("SDN-IP started");
+    }
+
+    @Deactivate
+    protected void deactivate() {
+        log.info("Stopped");
+    }
+}

+/**
+ * SDN-IP peering application.
+ */
+package org.onlab.onos.sdnip;
\ No newline at end of file

@@ -141,7 +141,7 @@ public class TopologyResource extends BaseResource {
     private ObjectNode json(ObjectMapper mapper, ElementId id, int group,
     private ObjectNode json(ObjectMapper mapper, ElementId id, int group,
                             String label, boolean isOnline) {
                             String label, boolean isOnline) {
         return mapper.createObjectNode()
         return mapper.createObjectNode()
-                .put("name", id.uri().toString())
+                .put("name", id.toString())
                 .put("label", label)
                 .put("label", label)
                 .put("group", group)
                 .put("group", group)
                 .put("online", isOnline);
                 .put("online", isOnline);
@@ -202,7 +202,7 @@ public class TopologyResource extends BaseResource {
     // Returns a formatted string for the element associated with the given
     // Returns a formatted string for the element associated with the given
     // connection point.
     // connection point.
     private static String id(ConnectPoint cp) {
     private static String id(ConnectPoint cp) {
-        return cp.elementId().uri().toString();
+        return cp.elementId().toString();
     }
     }
 
 
 }
 }

 /**
 /**
- * REST resources for the sample topology viewer application.
+ * Sample topology viewer application.
  */
  */
 package org.onlab.onos.tvue;
 package org.onlab.onos.tvue;

@@ -21,14 +21,14 @@ public final class Comparators {
     public static final Comparator<ElementId> ELEMENT_ID_COMPARATOR = new Comparator<ElementId>() {
     public static final Comparator<ElementId> ELEMENT_ID_COMPARATOR = new Comparator<ElementId>() {
         @Override
         @Override
         public int compare(ElementId id1, ElementId id2) {
         public int compare(ElementId id1, ElementId id2) {
-            return id1.uri().toString().compareTo(id2.uri().toString());
+            return id1.toString().compareTo(id2.toString());
         }
         }
     };
     };
 
 
     public static final Comparator<Element> ELEMENT_COMPARATOR = new Comparator<Element>() {
     public static final Comparator<Element> ELEMENT_COMPARATOR = new Comparator<Element>() {
         @Override
         @Override
         public int compare(Element e1, Element e2) {
         public int compare(Element e1, Element e2) {
-            return e1.id().uri().toString().compareTo(e2.id().uri().toString());
+            return e1.id().toString().compareTo(e2.id().toString());
         }
         }
     };
     };
 
 

@@ -22,8 +22,10 @@ public class SummaryCommand extends AbstractShellCommand {
     protected void execute() {
     protected void execute() {
         TopologyService topologyService = get(TopologyService.class);
         TopologyService topologyService = get(TopologyService.class);
         Topology topology = topologyService.currentTopology();
         Topology topology = topologyService.currentTopology();
-        print("version=%s, nodes=%d, devices=%d, links=%d, hosts=%d, clusters=%s, paths=%d, flows=%d, intents=%d",
-              get(CoreService.class).version().toString(),
+        print("node=%s, version=%s",
+              get(ClusterService.class).getLocalNode().ip(),
+              get(CoreService.class).version().toString());
+        print("nodes=%d, devices=%d, links=%d, hosts=%d, clusters=%s, paths=%d, flows=%d, intents=%d",
               get(ClusterService.class).getNodes().size(),
               get(ClusterService.class).getNodes().size(),
               get(DeviceService.class).getDeviceCount(),
               get(DeviceService.class).getDeviceCount(),
               get(LinkService.class).getLinkCount(),
               get(LinkService.class).getLinkCount(),

 package org.onlab.onos.cli.net;
 package org.onlab.onos.cli.net;
 
 
-import static com.google.common.collect.Lists.newArrayList;
-import static org.onlab.onos.cli.net.DevicesListCommand.getSortedDevices;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
+import com.google.common.collect.Maps;
 import org.apache.karaf.shell.commands.Argument;
 import org.apache.karaf.shell.commands.Argument;
 import org.apache.karaf.shell.commands.Command;
 import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.CoreService;
 import org.onlab.onos.cli.AbstractShellCommand;
 import org.onlab.onos.cli.AbstractShellCommand;
 import org.onlab.onos.cli.Comparators;
 import org.onlab.onos.cli.Comparators;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.Device;
@@ -18,37 +13,43 @@ import org.onlab.onos.net.flow.FlowEntry;
 import org.onlab.onos.net.flow.FlowEntry.FlowEntryState;
 import org.onlab.onos.net.flow.FlowEntry.FlowEntryState;
 import org.onlab.onos.net.flow.FlowRuleService;
 import org.onlab.onos.net.flow.FlowRuleService;
 
 
-import com.google.common.collect.Maps;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import static com.google.common.collect.Lists.newArrayList;
+import static org.onlab.onos.cli.net.DevicesListCommand.getSortedDevices;
 
 
 /**
 /**
  * Lists all currently-known hosts.
  * Lists all currently-known hosts.
  */
  */
 @Command(scope = "onos", name = "flows",
 @Command(scope = "onos", name = "flows",
-description = "Lists all currently-known flows.")
+         description = "Lists all currently-known flows.")
 public class FlowsListCommand extends AbstractShellCommand {
 public class FlowsListCommand extends AbstractShellCommand {
 
 
     public static final String ANY = "any";
     public static final String ANY = "any";
 
 
     private static final String FMT =
     private static final String FMT =
-            "   id=%s, state=%s, bytes=%s, packets=%s, duration=%s, priority=%s";
+            "   id=%s, state=%s, bytes=%s, packets=%s, duration=%s, priority=%s, appId=%s";
     private static final String TFMT = "      treatment=%s";
     private static final String TFMT = "      treatment=%s";
     private static final String SFMT = "      selector=%s";
     private static final String SFMT = "      selector=%s";
 
 
     @Argument(index = 1, name = "uri", description = "Device ID",
     @Argument(index = 1, name = "uri", description = "Device ID",
-            required = false, multiValued = false)
+              required = false, multiValued = false)
     String uri = null;
     String uri = null;
 
 
     @Argument(index = 0, name = "state", description = "Flow Rule state",
     @Argument(index = 0, name = "state", description = "Flow Rule state",
-            required = false, multiValued = false)
+              required = false, multiValued = false)
     String state = null;
     String state = null;
 
 
     @Override
     @Override
     protected void execute() {
     protected void execute() {
+        CoreService coreService = get(CoreService.class);
         DeviceService deviceService = get(DeviceService.class);
         DeviceService deviceService = get(DeviceService.class);
         FlowRuleService service = get(FlowRuleService.class);
         FlowRuleService service = get(FlowRuleService.class);
         Map<Device, List<FlowEntry>> flows = getSortedFlows(deviceService, service);
         Map<Device, List<FlowEntry>> flows = getSortedFlows(deviceService, service);
         for (Device d : getSortedDevices(deviceService)) {
         for (Device d : getSortedDevices(deviceService)) {
-            printFlows(d, flows.get(d));
+            printFlows(d, flows.get(d), coreService);
         }
         }
     }
     }
 
 
@@ -67,7 +68,7 @@ public class FlowsListCommand extends AbstractShellCommand {
             s = FlowEntryState.valueOf(state.toUpperCase());
             s = FlowEntryState.valueOf(state.toUpperCase());
         }
         }
         Iterable<Device> devices = uri == null ? deviceService.getDevices() :
         Iterable<Device> devices = uri == null ? deviceService.getDevices() :
-            Collections.singletonList(deviceService.getDevice(DeviceId.deviceId(uri)));
+                Collections.singletonList(deviceService.getDevice(DeviceId.deviceId(uri)));
         for (Device d : devices) {
         for (Device d : devices) {
             if (s == null) {
             if (s == null) {
                 rules = newArrayList(service.getFlowEntries(d.id()));
                 rules = newArrayList(service.getFlowEntries(d.id()));
@@ -87,16 +88,19 @@ public class FlowsListCommand extends AbstractShellCommand {
 
 
     /**
     /**
      * Prints flows.
      * Prints flows.
-     * @param d the device
+     *
+     * @param d     the device
      * @param flows the set of flows for that device.
      * @param flows the set of flows for that device.
      */
      */
-    protected void printFlows(Device d, List<FlowEntry> flows) {
+    protected void printFlows(Device d, List<FlowEntry> flows,
+                              CoreService coreService) {
         boolean empty = flows == null || flows.isEmpty();
         boolean empty = flows == null || flows.isEmpty();
         print("deviceId=%s, flowRuleCount=%d", d.id(), empty ? 0 : flows.size());
         print("deviceId=%s, flowRuleCount=%d", d.id(), empty ? 0 : flows.size());
         if (!empty) {
         if (!empty) {
             for (FlowEntry f : flows) {
             for (FlowEntry f : flows) {
-                print(FMT, Long.toHexString(f.id().value()), f.state(), f.bytes(),
-                      f.packets(), f.life(), f.priority());
+                print(FMT, Long.toHexString(f.id().value()), f.state(),
+                      f.bytes(), f.packets(), f.life(), f.priority(),
+                      coreService.getAppId(f.appId()).name());
                 print(SFMT, f.selector().criteria());
                 print(SFMT, f.selector().criteria());
                 print(TFMT, f.treatment().instructions());
                 print(TFMT, f.treatment().instructions());
             }
             }

+package org.onlab.onos.cli.net;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cli.AbstractShellCommand;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.flow.DefaultTrafficSelector;
+import org.onlab.onos.net.flow.DefaultTrafficTreatment;
+import org.onlab.onos.net.flow.TrafficSelector;
+import org.onlab.onos.net.flow.TrafficTreatment;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentEvent;
+import org.onlab.onos.net.intent.IntentEvent.Type;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.IntentListener;
+import org.onlab.onos.net.intent.IntentService;
+import org.onlab.onos.net.intent.PointToPointIntent;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.MacAddress;
+
+/**
+ * Installs point-to-point connectivity intents.
+ */
+@Command(scope = "onos", name = "push-test-intents",
+         description = "Installs random intents to test throughput")
+public class IntentPushTestCommand extends AbstractShellCommand
+    implements IntentListener {
+
+    @Argument(index = 0, name = "ingressDevice",
+              description = "Ingress Device/Port Description",
+              required = true, multiValued = false)
+    String ingressDeviceString = null;
+
+    @Argument(index = 1, name = "egressDevice",
+              description = "Egress Device/Port Description",
+              required = true, multiValued = false)
+    String egressDeviceString = null;
+
+    @Argument(index = 2, name = "count",
+            description = "Number of intents to push",
+            required = true, multiValued = false)
+    String countString = null;
+
+
+    private static long id = 0x7870001;
+
+    private IntentService service;
+    private CountDownLatch latch;
+    private long start, end;
+
+    @Override
+    protected void execute() {
+        service = get(IntentService.class);
+
+        DeviceId ingressDeviceId = DeviceId.deviceId(getDeviceId(ingressDeviceString));
+        PortNumber ingressPortNumber =
+                PortNumber.portNumber(getPortNumber(ingressDeviceString));
+        ConnectPoint ingress = new ConnectPoint(ingressDeviceId, ingressPortNumber);
+
+        DeviceId egressDeviceId = DeviceId.deviceId(getDeviceId(egressDeviceString));
+        PortNumber egressPortNumber =
+                PortNumber.portNumber(getPortNumber(egressDeviceString));
+        ConnectPoint egress = new ConnectPoint(egressDeviceId, egressPortNumber);
+
+        TrafficSelector.Builder selector = DefaultTrafficSelector.builder()
+                .matchEthType(Ethernet.TYPE_IPV4);
+        TrafficTreatment treatment = DefaultTrafficTreatment.builder().build();
+
+        int count = Integer.parseInt(countString);
+
+        service.addListener(this);
+        latch = new CountDownLatch(count);
+
+        start = System.currentTimeMillis();
+        for (int i = 0; i < count; i++) {
+            TrafficSelector s = selector
+                                .matchEthSrc(MacAddress.valueOf(i))
+                                .build();
+            Intent intent =
+                    new PointToPointIntent(new IntentId(id++),
+                                                     s,
+                                                     treatment,
+                                                     ingress,
+                                                     egress);
+            service.submit(intent);
+        }
+        try {
+            latch.await(5, TimeUnit.SECONDS);
+            printResults(count);
+        } catch (InterruptedException e) {
+            print(e.toString());
+        }
+        service.removeListener(this);
+    }
+
+    private void printResults(int count) {
+        long delta = end - start;
+        print("Time to install %d intents: %d ms", count, delta);
+    }
+
+    /**
+     * Extracts the port number portion of the ConnectPoint.
+     *
+     * @param deviceString string representing the device/port
+     * @return port number as a string, empty string if the port is not found
+     */
+    private String getPortNumber(String deviceString) {
+        int slash = deviceString.indexOf('/');
+        if (slash <= 0) {
+            return "";
+        }
+        return deviceString.substring(slash + 1, deviceString.length());
+    }
+
+    /**
+     * Extracts the device ID portion of the ConnectPoint.
+     *
+     * @param deviceString string representing the device/port
+     * @return device ID string
+     */
+    private String getDeviceId(String deviceString) {
+        int slash = deviceString.indexOf('/');
+        if (slash <= 0) {
+            return "";
+        }
+        return deviceString.substring(0, slash);
+    }
+
+    @Override
+    public void event(IntentEvent event) {
+        if (event.type() == Type.INSTALLED) {
+            end = event.time();
+            if (latch != null) {
+                latch.countDown();
+            } else {
+                log.warn("install event latch is null");
+            }
+        }
+    }
+}

@@ -19,17 +19,16 @@ import org.onlab.onos.net.intent.IntentState;
          description = "Wipes-out the entire network information base, i.e. devices, links, hosts")
          description = "Wipes-out the entire network information base, i.e. devices, links, hosts")
 public class WipeOutCommand extends ClustersListCommand {
 public class WipeOutCommand extends ClustersListCommand {
 
 
-    private static final String DISCLAIMER = "Delete everything please.";
+    private static final String PLEASE = "please";
 
 
-    @Argument(index = 0, name = "disclaimer", description = "Device ID",
+    @Argument(index = 0, name = "please", description = "Confirmation phrase",
               required = false, multiValued = false)
               required = false, multiValued = false)
-    String disclaimer = null;
+    String please = null;
 
 
     @Override
     @Override
     protected void execute() {
     protected void execute() {
-        if (disclaimer == null || !disclaimer.equals(DISCLAIMER)) {
-            print("I'm afraid I can't do that!\nPlease acknowledge with phrase: '%s'",
-                  DISCLAIMER);
+        if (please == null || !please.equals(PLEASE)) {
+            print("I'm afraid I can't do that!\nSay: %s", PLEASE);
             return;
             return;
         }
         }
 
 

@@ -82,6 +82,13 @@
                 <ref component-id="connectPointCompleter"/>
                 <ref component-id="connectPointCompleter"/>
             </completers>
             </completers>
         </command>
         </command>
+        <command>
+            <action class="org.onlab.onos.cli.net.IntentPushTestCommand"/>
+            <completers>
+                <ref component-id="connectPointCompleter"/>
+                <ref component-id="connectPointCompleter"/>
+            </completers>
+        </command>
 
 
         <command>
         <command>
             <action class="org.onlab.onos.cli.net.ClustersListCommand"/>
             <action class="org.onlab.onos.cli.net.ClustersListCommand"/>

 package org.onlab.onos;
 package org.onlab.onos;
 
 
-import java.util.Objects;
-import java.util.concurrent.atomic.AtomicInteger;
 
 
 /**
 /**
- * Application id generator class.
+ * Application identifier.
  */
  */
-public final class ApplicationId {
+public interface ApplicationId {
 
 
-    private static final AtomicInteger ID_DISPENCER = new AtomicInteger(1);
-    private final Integer id;
-
-    // Ban public construction
-    private ApplicationId(Integer id) {
-        this.id = id;
-    }
-
-    public Integer id() {
-        return id;
-    }
-
-    public static ApplicationId valueOf(Integer id) {
-        return new ApplicationId(id);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(id);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-        if (obj == null) {
-            return false;
-        }
-        if (!(obj instanceof ApplicationId)) {
-            return false;
-        }
-        ApplicationId other = (ApplicationId) obj;
-        return Objects.equals(this.id, other.id);
-    }
+    /**
+     * Returns the application id.
+     * @return a short value
+     */
+    short id();
 
 
     /**
     /**
-     * Returns a new application id.
-     *
-     * @return app id
+     * Returns the applications supplied identifier.
+     * @return a string identifier
      */
      */
-    public static ApplicationId getAppId() {
-        return new ApplicationId(ApplicationId.ID_DISPENCER.getAndIncrement());
-    }
+    String name();
 
 
 }
 }

@@ -12,4 +12,21 @@ public interface CoreService {
      */
      */
     Version version();
     Version version();
 
 
+    /**
+     * Registers a new application by its name, which is expected
+     * to follow the reverse DNS convention, e.g.
+     * {@code org.flying.circus.app}
+     *
+     * @param identifier string identifier
+     * @return the application id
+     */
+    ApplicationId registerApplication(String identifier);
+
+    /**
+     * Returns an existing application id from a given id.
+     * @param id the short value of the id
+     * @return an application id
+     */
+    ApplicationId getAppId(Short id);
+
 }
 }

+package org.onlab.onos.cluster;
+
+import com.google.common.base.Function;
+
+/**
+ * Function to convert ControllerNode to NodeId.
+ */
+public final class ControllerNodeToNodeId
+    implements Function<ControllerNode, NodeId> {
+
+    private static final ControllerNodeToNodeId INSTANCE = new ControllerNodeToNodeId();
+
+    @Override
+    public NodeId apply(ControllerNode input) {
+        return input.id();
+    }
+
+    /**
+     * Returns a Function to convert ControllerNode to NodeId.
+     *
+     * @return ControllerNodeToNodeId instance.
+     */
+    public static ControllerNodeToNodeId toNodeId() {
+        return INSTANCE;
+    }
+}

 package org.onlab.onos.net;
 package org.onlab.onos.net;
 
 
 import java.net.URI;
 import java.net.URI;
+import java.util.Objects;
 
 
 /**
 /**
  * Immutable representation of a device identity.
  * Immutable representation of a device identity.
  */
  */
 public final class DeviceId extends ElementId {
 public final class DeviceId extends ElementId {
 
 
+    /**
+     * Represents either no device, or an unspecified device.
+     */
+    public static final DeviceId NONE = deviceId("none:none");
+
+    private final URI uri;
+    private final String str;
+
     // Public construction is prohibited
     // Public construction is prohibited
     private DeviceId(URI uri) {
     private DeviceId(URI uri) {
-        super(uri);
+        this.uri = uri;
+        this.str = uri.toString();
+    }
+
+
+    // Default constructor for serialization
+    protected DeviceId() {
+        this.uri = null;
+        this.str = null;
     }
     }
 
 
     /**
     /**
@@ -30,4 +47,36 @@ public final class DeviceId extends ElementId {
         return deviceId(URI.create(string));
         return deviceId(URI.create(string));
     }
     }
 
 
+    /**
+     * Returns the backing URI.
+     *
+     * @return backing URI
+     */
+    public URI uri() {
+        return uri;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(str);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof DeviceId) {
+            final DeviceId that = (DeviceId) obj;
+            return this.getClass() == that.getClass() &&
+                    Objects.equals(this.str, that.str);
+        }
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return str;
+    }
+
 }
 }

 package org.onlab.onos.net;
 package org.onlab.onos.net;
 
 
-import java.net.URI;
-import java.util.Objects;
-
 /**
 /**
  * Immutable representation of a network element identity.
  * Immutable representation of a network element identity.
  */
  */
 public abstract class ElementId {
 public abstract class ElementId {
-
-    private final URI uri;
-
-    // Default constructor for serialization
-    protected ElementId() {
-        this.uri = null;
-    }
-
-    /**
-     * Creates an element identifier using the supplied URI.
-     *
-     * @param uri backing URI
-     */
-    protected ElementId(URI uri) {
-        this.uri = uri;
-    }
-
-    /**
-     * Returns the backing URI.
-     *
-     * @return backing URI
-     */
-    public URI uri() {
-        return uri;
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(uri);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-        if (obj instanceof ElementId) {
-            final ElementId that = (ElementId) obj;
-            return this.getClass() == that.getClass() &&
-                    Objects.equals(this.uri, that.uri);
-        }
-        return false;
-    }
-
-    @Override
-    public String toString() {
-        return uri.toString();
-    }
-
 }
 }

@@ -3,44 +3,69 @@ package org.onlab.onos.net;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.onlab.packet.VlanId;
 
 
-import java.net.URI;
+import java.util.Objects;
+
+import static com.google.common.base.Preconditions.checkArgument;
 
 
 /**
 /**
  * Immutable representation of a host identity.
  * Immutable representation of a host identity.
  */
  */
 public final class HostId extends ElementId {
 public final class HostId extends ElementId {
 
 
-    private static final String NIC = "nic";
-
     /**
     /**
      * Represents either no host, or an unspecified host; used for creating
      * Represents either no host, or an unspecified host; used for creating
      * open ingress/egress edge links.
      * open ingress/egress edge links.
      */
      */
-    public static final HostId NONE = hostId(NIC + ":none-0");
+    public static final HostId NONE = new HostId(MacAddress.ZERO, VlanId.NONE);
+
+    private static final int MAC_LENGTH = 17;
+    private static final int MIN_ID_LENGTH = 19;
+
+    private final MacAddress mac;
+    private final VlanId vlanId;
 
 
     // Public construction is prohibited
     // Public construction is prohibited
-    private HostId(URI uri) {
-        super(uri);
+    private HostId(MacAddress mac, VlanId vlanId) {
+        this.mac = mac;
+        this.vlanId = vlanId;
+    }
+
+    // Default constructor for serialization
+    private HostId() {
+        this.mac = null;
+        this.vlanId = null;
     }
     }
 
 
     /**
     /**
-     * Creates a device id using the supplied URI.
+     * Returns the host MAC address.
      *
      *
-     * @param uri device URI
-     * @return host identifier
+     * @return MAC address
+     */
+    public MacAddress mac() {
+        return mac;
+    }
+
+    /**
+     * Returns the host MAC address.
+     *
+     * @return MAC address
      */
      */
-    public static HostId hostId(URI uri) {
-        return new HostId(uri);
+    public VlanId vlanId() {
+        return vlanId;
     }
     }
 
 
     /**
     /**
-     * Creates a device id using the supplied URI string.
+     * Creates a device id using the supplied ID string.
      *
      *
      * @param string device URI string
      * @param string device URI string
      * @return host identifier
      * @return host identifier
      */
      */
     public static HostId hostId(String string) {
     public static HostId hostId(String string) {
-        return hostId(URI.create(string));
+        checkArgument(string.length() >= MIN_ID_LENGTH,
+                      "Host ID must be at least %s characters", MIN_ID_LENGTH);
+        MacAddress mac = MacAddress.valueOf(string.substring(0, MAC_LENGTH));
+        VlanId vlanId = VlanId.vlanId(Short.parseShort(string.substring(MAC_LENGTH + 1)));
+        return new HostId(mac, vlanId);
     }
     }
 
 
     /**
     /**
@@ -51,7 +76,7 @@ public final class HostId extends ElementId {
      * @return host identifier
      * @return host identifier
      */
      */
     public static HostId hostId(MacAddress mac, VlanId vlanId) {
     public static HostId hostId(MacAddress mac, VlanId vlanId) {
-        return hostId(NIC + ":" + mac + "-" + vlanId);
+        return new HostId(mac, vlanId);
     }
     }
 
 
     /**
     /**
@@ -64,4 +89,26 @@ public final class HostId extends ElementId {
         return hostId(mac, VlanId.vlanId(VlanId.UNTAGGED));
         return hostId(mac, VlanId.vlanId(VlanId.UNTAGGED));
     }
     }
 
 
+    public String toString() {
+        return mac + "/" + vlanId;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(mac, vlanId);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof HostId) {
+            final HostId other = (HostId) obj;
+            return Objects.equals(this.mac, other.mac) &&
+                    Objects.equals(this.vlanId, other.vlanId);
+        }
+        return false;
+    }
+
 }
 }

 package org.onlab.onos.net;
 package org.onlab.onos.net;
 
 
+import static org.onlab.onos.net.PortNumber.P0;
+
 /**
 /**
  * Representation of a network edge location where an end-station host is
  * Representation of a network edge location where an end-station host is
  * connected.
  * connected.
  */
  */
 public class HostLocation extends ConnectPoint {
 public class HostLocation extends ConnectPoint {
 
 
+    /**
+     * Represents a no location or an unknown location.
+     */
+    public static final HostLocation NONE = new HostLocation(DeviceId.NONE, P0, 0L);
+
     // Note that time is explicitly excluded from the notion of equality.
     // Note that time is explicitly excluded from the notion of equality.
     private final long time;
     private final long time;
 
 

+package org.onlab.onos.net;
+
+/**
+ * Representation of a network resource.
+ */
+public interface NetworkResource {
+}

@@ -4,6 +4,8 @@ import org.onlab.onos.net.AbstractDescription;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.SparseAnnotations;
 import org.onlab.onos.net.SparseAnnotations;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Default implementation of immutable port description.
  * Default implementation of immutable port description.
  */
  */
@@ -48,6 +50,15 @@ public class DefaultPortDescription extends AbstractDescription
         return isEnabled;
         return isEnabled;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("number", number)
+                .add("isEnabled", isEnabled)
+                .add("annotations", annotations())
+                .toString();
+    }
+
     // default constructor for serialization
     // default constructor for serialization
     private DefaultPortDescription() {
     private DefaultPortDescription() {
         this.number = null;
         this.number = null;

+package org.onlab.onos.net.flow;
+
+import java.util.List;
+
+/**
+ * Interface capturing the result of a batch operation.
+ *
+ */
+public interface BatchOperationResult<T> {
+
+    /**
+     * Returns whether the operation was successful.
+     * @return true if successful, false otherwise
+     */
+    boolean isSuccess();
+
+    /**
+     * Obtains a list of items which failed.
+     * @return a list of failures
+     */
+    List<T> failedItems();
+
+}

 package org.onlab.onos.net.flow;
 package org.onlab.onos.net.flow;
 
 
-public class CompletedBatchOperation {
+import java.util.List;
+
+import com.google.common.collect.ImmutableList;
+
+public class CompletedBatchOperation implements BatchOperationResult<FlowEntry> {
+
+
+    private final boolean success;
+    private final List<FlowEntry> failures;
+
+    public CompletedBatchOperation(boolean success, List<FlowEntry> failures) {
+        this.success = success;
+        this.failures = ImmutableList.copyOf(failures);
+    }
+
+    @Override
+    public boolean isSuccess() {
+        return success;
+    }
+
+    @Override
+    public List<FlowEntry> failedItems() {
+        return failures;
+    }
 
 
 
 
 }
 }

@@ -17,6 +17,10 @@ public class DefaultFlowEntry extends DefaultFlowRule implements FlowEntry {
 
 
     private long lastSeen = -1;
     private long lastSeen = -1;
 
 
+    private final int errType;
+
+    private final int errCode;
+
 
 
     public DefaultFlowEntry(DeviceId deviceId, TrafficSelector selector,
     public DefaultFlowEntry(DeviceId deviceId, TrafficSelector selector,
             TrafficTreatment treatment, int priority, FlowEntryState state,
             TrafficTreatment treatment, int priority, FlowEntryState state,
@@ -27,6 +31,8 @@ public class DefaultFlowEntry extends DefaultFlowRule implements FlowEntry {
         this.life = life;
         this.life = life;
         this.packets = packets;
         this.packets = packets;
         this.bytes = bytes;
         this.bytes = bytes;
+        this.errCode = -1;
+        this.errType = -1;
         this.lastSeen = System.currentTimeMillis();
         this.lastSeen = System.currentTimeMillis();
     }
     }
 
 
@@ -37,6 +43,8 @@ public class DefaultFlowEntry extends DefaultFlowRule implements FlowEntry {
         this.life = life;
         this.life = life;
         this.packets = packets;
         this.packets = packets;
         this.bytes = bytes;
         this.bytes = bytes;
+        this.errCode = -1;
+        this.errType = -1;
         this.lastSeen = System.currentTimeMillis();
         this.lastSeen = System.currentTimeMillis();
     }
     }
 
 
@@ -46,9 +54,18 @@ public class DefaultFlowEntry extends DefaultFlowRule implements FlowEntry {
         this.life = 0;
         this.life = 0;
         this.packets = 0;
         this.packets = 0;
         this.bytes = 0;
         this.bytes = 0;
+        this.errCode = -1;
+        this.errType = -1;
         this.lastSeen = System.currentTimeMillis();
         this.lastSeen = System.currentTimeMillis();
     }
     }
 
 
+    public DefaultFlowEntry(FlowRule rule, int errType, int errCode) {
+        super(rule);
+        this.state = FlowEntryState.FAILED;
+        this.errType = errType;
+        this.errCode = errCode;
+    }
+
     @Override
     @Override
     public long life() {
     public long life() {
         return life;
         return life;
@@ -100,6 +117,16 @@ public class DefaultFlowEntry extends DefaultFlowRule implements FlowEntry {
     }
     }
 
 
     @Override
     @Override
+    public int errType() {
+        return this.errType;
+    }
+
+    @Override
+    public int errCode() {
+        return this.errCode;
+    }
+
+    @Override
     public String toString() {
     public String toString() {
         return toStringHelper(this)
         return toStringHelper(this)
                 .add("rule", super.toString())
                 .add("rule", super.toString())
@@ -108,4 +135,6 @@ public class DefaultFlowEntry extends DefaultFlowRule implements FlowEntry {
     }
     }
 
 
 
 
+
+
 }
 }

@@ -21,7 +21,7 @@ public class DefaultFlowRule implements FlowRule {
 
 
     private final FlowId id;
     private final FlowId id;
 
 
-    private final ApplicationId appId;
+    private final short appId;
 
 
     private final int timeout;
     private final int timeout;
 
 
@@ -36,7 +36,7 @@ public class DefaultFlowRule implements FlowRule {
         this.timeout = timeout;
         this.timeout = timeout;
         this.created = System.currentTimeMillis();
         this.created = System.currentTimeMillis();
 
 
-        this.appId = ApplicationId.valueOf((int) (flowId >> 32));
+        this.appId = (short) (flowId >>> 48);
         this.id = FlowId.valueOf(flowId);
         this.id = FlowId.valueOf(flowId);
     }
     }
 
 
@@ -52,11 +52,11 @@ public class DefaultFlowRule implements FlowRule {
         this.priority = priority;
         this.priority = priority;
         this.selector = selector;
         this.selector = selector;
         this.treatment = treatement;
         this.treatment = treatement;
-        this.appId = appId;
+        this.appId = appId.id();
         this.timeout = timeout;
         this.timeout = timeout;
         this.created = System.currentTimeMillis();
         this.created = System.currentTimeMillis();
 
 
-        this.id = FlowId.valueOf((((long) appId().id()) << 32) | (this.hash() & 0xffffffffL));
+        this.id = FlowId.valueOf((((long) this.appId) << 48) | (this.hash() & 0x0000ffffffffL));
     }
     }
 
 
     public DefaultFlowRule(FlowRule rule) {
     public DefaultFlowRule(FlowRule rule) {
@@ -78,7 +78,7 @@ public class DefaultFlowRule implements FlowRule {
     }
     }
 
 
     @Override
     @Override
-    public ApplicationId appId() {
+    public short appId() {
         return appId;
         return appId;
     }
     }
 
 

@@ -140,6 +140,16 @@ public final class DefaultTrafficSelector implements TrafficSelector {
         }
         }
 
 
         @Override
         @Override
+        public Builder matchTcpSrc(Short tcpPort) {
+            return add(Criteria.matchTcpSrc(tcpPort));
+        }
+
+        @Override
+        public Builder matchTcpDst(Short tcpPort) {
+            return add(Criteria.matchTcpDst(tcpPort));
+        }
+
+        @Override
         public TrafficSelector build() {
         public TrafficSelector build() {
             return new DefaultTrafficSelector(ImmutableSet.copyOf(selector.values()));
             return new DefaultTrafficSelector(ImmutableSet.copyOf(selector.values()));
         }
         }

 package org.onlab.onos.net.flow;
 package org.onlab.onos.net.flow;
 
 
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.Collections;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Objects;
-
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.flow.instructions.Instruction;
 import org.onlab.onos.net.flow.instructions.Instruction;
 import org.onlab.onos.net.flow.instructions.Instructions;
 import org.onlab.onos.net.flow.instructions.Instructions;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.onlab.packet.VlanId;
-import org.slf4j.Logger;
+
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
 
 
 /**
 /**
  * Default traffic treatment implementation.
  * Default traffic treatment implementation.
@@ -58,7 +55,7 @@ public final class DefaultTrafficTreatment implements TrafficTreatment {
         }
         }
         if (obj instanceof DefaultTrafficTreatment) {
         if (obj instanceof DefaultTrafficTreatment) {
             DefaultTrafficTreatment that = (DefaultTrafficTreatment) obj;
             DefaultTrafficTreatment that = (DefaultTrafficTreatment) obj;
-            return  Objects.equals(instructions, that.instructions);
+            return Objects.equals(instructions, that.instructions);
 
 
         }
         }
         return false;
         return false;
@@ -70,8 +67,6 @@ public final class DefaultTrafficTreatment implements TrafficTreatment {
      */
      */
     public static final class Builder implements TrafficTreatment.Builder {
     public static final class Builder implements TrafficTreatment.Builder {
 
 
-        private final Logger log = getLogger(getClass());
-
         boolean drop = false;
         boolean drop = false;
 
 
         List<Instruction> outputs = new LinkedList<>();
         List<Instruction> outputs = new LinkedList<>();
@@ -107,7 +102,8 @@ public final class DefaultTrafficTreatment implements TrafficTreatment {
                     groups.add(instruction);
                     groups.add(instruction);
                     break;
                     break;
                 default:
                 default:
-                    log.warn("Unknown instruction type {}", instruction.type());
+                    throw new IllegalArgumentException("Unknown instruction type: " +
+                                                               instruction.type());
             }
             }
             return this;
             return this;
         }
         }

@@ -29,7 +29,12 @@ public interface FlowEntry extends FlowRule {
         /**
         /**
          * Flow has been removed from flow table and can be purged.
          * Flow has been removed from flow table and can be purged.
          */
          */
-        REMOVED
+        REMOVED,
+
+        /**
+         * Indicates that the installation of this flow has failed.
+         */
+        FAILED
     }
     }
 
 
     /**
     /**
@@ -95,4 +100,16 @@ public interface FlowEntry extends FlowRule {
      */
      */
     void setBytes(long bytes);
     void setBytes(long bytes);
 
 
+    /**
+     * Indicates the error type.
+     * @return an integer value of the error
+     */
+    int errType();
+
+    /**
+     * Indicates the error code.
+     * @return an integer value of the error
+     */
+    int errCode();
+
 }
 }

 package org.onlab.onos.net.flow;
 package org.onlab.onos.net.flow;
 
 
-import org.onlab.onos.ApplicationId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.intent.BatchOperationTarget;
 import org.onlab.onos.net.intent.BatchOperationTarget;
 
 
@@ -26,7 +25,7 @@ public interface FlowRule extends BatchOperationTarget {
      *
      *
      * @return an applicationId
      * @return an applicationId
      */
      */
-    ApplicationId appId();
+    short appId();
 
 
     /**
     /**
      * Returns the flow rule priority given in natural order; higher numbers
      * Returns the flow rule priority given in natural order; higher numbers

@@ -37,6 +37,12 @@ public interface FlowRuleProvider extends Provider {
      */
      */
     void removeRulesById(ApplicationId id, FlowRule... flowRules);
     void removeRulesById(ApplicationId id, FlowRule... flowRules);
 
 
-    Future<Void> executeBatch(BatchOperation<FlowRuleBatchEntry> batch);
+    /**
+     * Installs a batch of flow rules. Each flowrule is associated to an
+     * operation which results in either addition, removal or modification.
+     * @param batch a batch of flow rules
+     * @return a future indicating the status of this execution
+     */
+    Future<CompletedBatchOperation> executeBatch(BatchOperation<FlowRuleBatchEntry> batch);
 
 
 }
 }

@@ -98,6 +98,20 @@ public interface TrafficSelector {
         public Builder matchIPDst(IpPrefix ip);
         public Builder matchIPDst(IpPrefix ip);
 
 
         /**
         /**
+         * Matches a TCP source port number.
+         * @param tcpPort a TCP source port number
+         * @return a selection builder
+         */
+        public Builder matchTcpSrc(Short tcpPort);
+
+        /**
+         * Matches a TCP destination port number.
+         * @param tcpPort a TCP destination port number
+         * @return a selection builder
+         */
+        public Builder matchTcpDst(Short tcpPort);
+
+        /**
          * Builds an immutable traffic selector.
          * Builds an immutable traffic selector.
          *
          *
          * @return traffic selector
          * @return traffic selector

@@ -113,6 +113,25 @@ public final class Criteria {
         return new IPCriterion(ip, Type.IPV4_DST);
         return new IPCriterion(ip, Type.IPV4_DST);
     }
     }
 
 
+    /**
+     * Creates a match on TCP source port field using the specified value.
+     *
+     * @param tcpPort
+     * @return match criterion
+     */
+    public static Criterion matchTcpSrc(Short tcpPort) {
+        return new TcpPortCriterion(tcpPort, Type.TCP_SRC);
+    }
+
+    /**
+     * Creates a match on TCP destination port field using the specified value.
+     *
+     * @param tcpPort
+     * @return match criterion
+     */
+    public static Criterion matchTcpDst(Short tcpPort) {
+        return new TcpPortCriterion(tcpPort, Type.TCP_DST);
+    }
 
 
     /*
     /*
      * Implementations of criteria.
      * Implementations of criteria.
@@ -437,4 +456,49 @@ public final class Criteria {
     }
     }
 
 
 
 
+    public static final class TcpPortCriterion implements Criterion {
+
+        private final Short tcpPort;
+        private final Type type;
+
+        public TcpPortCriterion(Short tcpPort, Type type) {
+            this.tcpPort = tcpPort;
+            this.type = type;
+        }
+
+        @Override
+        public Type type() {
+            return this.type;
+        }
+
+        public Short tcpPort() {
+            return this.tcpPort;
+        }
+
+        @Override
+        public String toString() {
+            return toStringHelper(type().toString())
+                    .add("tcpPort", tcpPort).toString();
+        }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(tcpPort, type);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj instanceof TcpPortCriterion) {
+                TcpPortCriterion that = (TcpPortCriterion) obj;
+                return Objects.equals(tcpPort, that.tcpPort) &&
+                        Objects.equals(type, that.type);
+
+
+            }
+            return false;
+        }
+    }
 }
 }

 package org.onlab.onos.net.host;
 package org.onlab.onos.net.host;
 
 
-import com.google.common.collect.ImmutableSet;
 import org.onlab.onos.net.AbstractDescription;
 import org.onlab.onos.net.AbstractDescription;
 import org.onlab.onos.net.HostLocation;
 import org.onlab.onos.net.HostLocation;
 import org.onlab.onos.net.SparseAnnotations;
 import org.onlab.onos.net.SparseAnnotations;
@@ -8,9 +7,6 @@ import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.onlab.packet.VlanId;
 
 
-import java.util.HashSet;
-import java.util.Set;
-
 import static com.google.common.base.MoreObjects.toStringHelper;
 import static com.google.common.base.MoreObjects.toStringHelper;
 
 
 /**
 /**
@@ -22,7 +18,7 @@ public class DefaultHostDescription extends AbstractDescription
     private final MacAddress mac;
     private final MacAddress mac;
     private final VlanId vlan;
     private final VlanId vlan;
     private final HostLocation location;
     private final HostLocation location;
-    private final Set<IpPrefix> ips;
+    private final IpPrefix ip;
 
 
     /**
     /**
      * Creates a host description using the supplied information.
      * Creates a host description using the supplied information.
@@ -35,7 +31,7 @@ public class DefaultHostDescription extends AbstractDescription
     public DefaultHostDescription(MacAddress mac, VlanId vlan,
     public DefaultHostDescription(MacAddress mac, VlanId vlan,
                                   HostLocation location,
                                   HostLocation location,
                                   SparseAnnotations... annotations) {
                                   SparseAnnotations... annotations) {
-        this(mac, vlan, location, new HashSet<IpPrefix>(), annotations);
+        this(mac, vlan, location, null, annotations);
     }
     }
 
 
     /**
     /**
@@ -44,17 +40,17 @@ public class DefaultHostDescription extends AbstractDescription
      * @param mac         host MAC address
      * @param mac         host MAC address
      * @param vlan        host VLAN identifier
      * @param vlan        host VLAN identifier
      * @param location    host location
      * @param location    host location
-     * @param ips         of host IP addresses
+     * @param ip          host IP address
      * @param annotations optional key/value annotations map
      * @param annotations optional key/value annotations map
      */
      */
     public DefaultHostDescription(MacAddress mac, VlanId vlan,
     public DefaultHostDescription(MacAddress mac, VlanId vlan,
-                                  HostLocation location, Set<IpPrefix> ips,
+                                  HostLocation location, IpPrefix ip,
                                   SparseAnnotations... annotations) {
                                   SparseAnnotations... annotations) {
         super(annotations);
         super(annotations);
         this.mac = mac;
         this.mac = mac;
         this.vlan = vlan;
         this.vlan = vlan;
         this.location = location;
         this.location = location;
-        this.ips = new HashSet<>(ips);
+        this.ip = ip;
     }
     }
 
 
     @Override
     @Override
@@ -73,8 +69,8 @@ public class DefaultHostDescription extends AbstractDescription
     }
     }
 
 
     @Override
     @Override
-    public Set<IpPrefix> ipAddresses() {
-        return ImmutableSet.copyOf(ips);
+    public IpPrefix ipAddress() {
+        return ip;
     }
     }
 
 
     @Override
     @Override
@@ -83,7 +79,7 @@ public class DefaultHostDescription extends AbstractDescription
                 .add("mac", mac)
                 .add("mac", mac)
                 .add("vlan", vlan)
                 .add("vlan", vlan)
                 .add("location", location)
                 .add("location", location)
-                .add("ipAddresses", ips)
+                .add("ipAddress", ip)
                 .toString();
                 .toString();
     }
     }
 
 

 package org.onlab.onos.net.host;
 package org.onlab.onos.net.host;
 
 
-import java.util.Set;
-
 import org.onlab.onos.net.Description;
 import org.onlab.onos.net.Description;
 import org.onlab.onos.net.HostLocation;
 import org.onlab.onos.net.HostLocation;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.IpPrefix;
@@ -35,10 +33,10 @@ public interface HostDescription extends Description {
     HostLocation location();
     HostLocation location();
 
 
     /**
     /**
-     * Returns zero or more IP address(es) associated with this host's MAC.
+     * Returns the IP address associated with this host's MAC.
      *
      *
-     * @return a set of IP addresses.
+     * @return host IP address
      */
      */
-    Set<IpPrefix> ipAddresses();
+    IpPrefix ipAddress();
 
 
 }
 }

 package org.onlab.onos.net.intent;
 package org.onlab.onos.net.intent;
 
 
+import java.util.concurrent.Future;
+
+import org.onlab.onos.net.flow.CompletedBatchOperation;
+
 /**
 /**
  * Abstraction of entity capable of installing intents to the environment.
  * Abstraction of entity capable of installing intents to the environment.
  */
  */
@@ -10,7 +14,7 @@ public interface IntentInstaller<T extends InstallableIntent> {
      * @param intent intent to be installed
      * @param intent intent to be installed
      * @throws IntentException if issues are encountered while installing the intent
      * @throws IntentException if issues are encountered while installing the intent
      */
      */
-    void install(T intent);
+    Future<CompletedBatchOperation> install(T intent);
 
 
     /**
     /**
      * Uninstalls the specified intent from the environment.
      * Uninstalls the specified intent from the environment.
@@ -18,5 +22,5 @@ public interface IntentInstaller<T extends InstallableIntent> {
      * @param intent intent to be uninstalled
      * @param intent intent to be uninstalled
      * @throws IntentException if issues are encountered while uninstalling the intent
      * @throws IntentException if issues are encountered while uninstalling the intent
      */
      */
-    void uninstall(T intent);
+    Future<CompletedBatchOperation> uninstall(T intent);
 }
 }

@@ -33,6 +33,8 @@ public interface IntentStore extends Store<IntentEvent, IntentStoreDelegate> {
 
 
     /**
     /**
      * Returns the number of intents in the store.
      * Returns the number of intents in the store.
+     *
+     * @return the number of intents in the store
      */
      */
     long getIntentCount();
     long getIntentCount();
 
 
@@ -44,7 +46,7 @@ public interface IntentStore extends Store<IntentEvent, IntentStoreDelegate> {
     Iterable<Intent> getIntents();
     Iterable<Intent> getIntents();
 
 
     /**
     /**
-     * Returns the intent with the specified identifer.
+     * Returns the intent with the specified identifier.
      *
      *
      * @param intentId intent identification
      * @param intentId intent identification
      * @return intent or null if not found
      * @return intent or null if not found
@@ -94,7 +96,6 @@ public interface IntentStore extends Store<IntentEvent, IntentStoreDelegate> {
      * specified original intent.
      * specified original intent.
      *
      *
      * @param intentId original intent identifier
      * @param intentId original intent identifier
-     * @return compiled state transition event
      */
      */
     void removeInstalledIntents(IntentId intentId);
     void removeInstalledIntents(IntentId intentId);
 
 

@@ -53,4 +53,4 @@
  * while the system determines where to perform the compilation or while it
  * while the system determines where to perform the compilation or while it
  * performs global recomputation/optimization across all prior intents.
  * performs global recomputation/optimization across all prior intents.
  */
  */
-package org.onlab.onos.net.intent;
\ No newline at end of file
+package org.onlab.onos.net.intent;

 package org.onlab.onos.store;
 package org.onlab.onos.store;
 
 
+import java.util.List;
+
 import org.onlab.onos.event.Event;
 import org.onlab.onos.event.Event;
 
 
 import static com.google.common.base.Preconditions.checkState;
 import static com.google.common.base.Preconditions.checkState;
@@ -41,4 +43,15 @@ public class AbstractStore<E extends Event, D extends StoreDelegate<E>>
             delegate.notify(event);
             delegate.notify(event);
         }
         }
     }
     }
+
+    /**
+     * Notifies the delegate with the specified list of events.
+     *
+     * @param events list of events to delegate
+     */
+    protected void notifyDelegate(List<E> events) {
+        for (E event: events) {
+            notifyDelegate(event);
+        }
+    }
 }
 }

+package org.onlab.onos;
+
+import java.util.Objects;
+
+/**
+ * Test application ID.
+ */
+public class TestApplicationId implements ApplicationId {
+
+    private final String name;
+    private final short id;
+
+    public TestApplicationId(String name) {
+        this.name = name;
+        this.id = (short) Objects.hash(name);
+    }
+
+    public static ApplicationId create(String name) {
+        return new TestApplicationId(name);
+    }
+
+    @Override
+    public short id() {
+        return id;
+    }
+
+    @Override
+    public String name() {
+        return name;
+    }
+}

+package org.onlab.onos.cluster;
+
+import static org.junit.Assert.*;
+import static org.onlab.onos.cluster.ControllerNodeToNodeId.toNodeId;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+import org.onlab.packet.IpPrefix;
+
+import com.google.common.collect.FluentIterable;
+
+
+public class ControllerNodeToNodeIdTest {
+
+    private static final NodeId NID1 = new NodeId("foo");
+    private static final NodeId NID2 = new NodeId("bar");
+    private static final NodeId NID3 = new NodeId("buz");
+
+    private static final IpPrefix IP1 = IpPrefix.valueOf("127.0.0.1");
+    private static final IpPrefix IP2 = IpPrefix.valueOf("127.0.0.2");
+    private static final IpPrefix IP3 = IpPrefix.valueOf("127.0.0.3");
+
+    private static final ControllerNode CN1 = new DefaultControllerNode(NID1, IP1);
+    private static final ControllerNode CN2 = new DefaultControllerNode(NID2, IP2);
+    private static final ControllerNode CN3 = new DefaultControllerNode(NID3, IP3);
+
+
+    @Test
+    public final void testToNodeId() {
+
+        final Iterable<ControllerNode> nodes = Arrays.asList(CN1, CN2, CN3);
+        final List<NodeId> nodeIds = Arrays.asList(NID1, NID2, NID3);
+
+        assertEquals(nodeIds,
+                FluentIterable.from(nodes)
+                    .transform(toNodeId())
+                    .toList());
+    }
+
+}

@@ -9,6 +9,7 @@ import static org.onlab.onos.event.TestEvent.Type.FOO;
 import java.util.List;
 import java.util.List;
 import java.util.Timer;
 import java.util.Timer;
 
 
+import org.junit.Ignore;
 import org.junit.Test;
 import org.junit.Test;
 
 
 /**
 /**
@@ -41,19 +42,23 @@ public class AbstractEventAccumulatorTest {
         assertEquals("incorrect batch", "abcde", accumulator.batch);
         assertEquals("incorrect batch", "abcde", accumulator.batch);
     }
     }
 
 
+    @Ignore("FIXME: timing sensitive test failing randomly.")
     @Test
     @Test
     public void timeTrigger() {
     public void timeTrigger() {
         TestAccumulator accumulator = new TestAccumulator();
         TestAccumulator accumulator = new TestAccumulator();
         accumulator.add(new TestEvent(FOO, "a"));
         accumulator.add(new TestEvent(FOO, "a"));
-        delay(40);
+        delay(30);
         assertTrue("should not have fired yet", accumulator.batch.isEmpty());
         assertTrue("should not have fired yet", accumulator.batch.isEmpty());
         accumulator.add(new TestEvent(FOO, "b"));
         accumulator.add(new TestEvent(FOO, "b"));
-        delay(40);
+        delay(30);
         assertTrue("should not have fired yet", accumulator.batch.isEmpty());
         assertTrue("should not have fired yet", accumulator.batch.isEmpty());
         accumulator.add(new TestEvent(FOO, "c"));
         accumulator.add(new TestEvent(FOO, "c"));
-        delay(40);
+        delay(30);
+        assertTrue("should not have fired yet", accumulator.batch.isEmpty());
+        accumulator.add(new TestEvent(FOO, "d"));
+        delay(30);
         assertFalse("should have fired", accumulator.batch.isEmpty());
         assertFalse("should have fired", accumulator.batch.isEmpty());
-        assertEquals("incorrect batch", "abc", accumulator.batch);
+        assertEquals("incorrect batch", "abcd", accumulator.batch);
     }
     }
 
 
     @Test
     @Test

@@ -18,8 +18,8 @@ public class DefaultEdgeLinkTest {
 
 
     private static final ProviderId PID = new ProviderId("of", "foo");
     private static final ProviderId PID = new ProviderId("of", "foo");
     private static final DeviceId DID1 = deviceId("of:foo");
     private static final DeviceId DID1 = deviceId("of:foo");
-    private static final HostId HID1 = hostId("nic:foobar");
-    private static final HostId HID2 = hostId("nic:barfoo");
+    private static final HostId HID1 = hostId("00:00:00:00:00:01/-1");
+    private static final HostId HID2 = hostId("00:00:00:00:00:01/-1");
     private static final PortNumber P0 = portNumber(0);
     private static final PortNumber P0 = portNumber(0);
     private static final PortNumber P1 = portNumber(1);
     private static final PortNumber P1 = portNumber(1);
 
 
@@ -35,12 +35,8 @@ public class DefaultEdgeLinkTest {
         EdgeLink l4 = new DefaultEdgeLink(PID, cp(HID2, P0),
         EdgeLink l4 = new DefaultEdgeLink(PID, cp(HID2, P0),
                                           new HostLocation(DID1, P1, 123L), false);
                                           new HostLocation(DID1, P1, 123L), false);
 
 
-        EdgeLink l5 = new DefaultEdgeLink(PID, cp(HID1, P0),
-                                          new HostLocation(DID1, P1, 123L), false);
-
         new EqualsTester().addEqualityGroup(l1, l2)
         new EqualsTester().addEqualityGroup(l1, l2)
                 .addEqualityGroup(l3, l4)
                 .addEqualityGroup(l3, l4)
-                .addEqualityGroup(l5)
                 .testEquals();
                 .testEquals();
     }
     }
 
 

@@ -8,7 +8,7 @@ import static org.onlab.onos.net.DeviceId.deviceId;
 /**
 /**
  * Test of the device identifier.
  * Test of the device identifier.
  */
  */
-public class DeviceIdTest extends ElementIdTest {
+public class DeviceIdTest {
 
 
     @Test
     @Test
     public void basics() {
     public void basics() {

-package org.onlab.onos.net;
-
-import com.google.common.testing.EqualsTester;
-import org.junit.Test;
-
-import java.net.URI;
-
-import static org.junit.Assert.assertEquals;
-
-/**
- * Test of the network element identifier.
- */
-public class ElementIdTest {
-
-    private static class FooId extends ElementId {
-        public FooId(URI uri) {
-            super(uri);
-        }
-    }
-
-    public static URI uri(String str) {
-        return URI.create(str);
-    }
-
-    @Test
-    public void basics() {
-        new EqualsTester()
-                .addEqualityGroup(new FooId(uri("of:foo")),
-                                  new FooId(uri("of:foo")))
-                .addEqualityGroup(new FooId(uri("of:bar")))
-                .testEquals();
-        assertEquals("wrong uri", uri("ofcfg:foo"),
-                     new FooId(uri("ofcfg:foo")).uri());
-    }
-
-}

@@ -11,20 +11,18 @@ import static org.onlab.onos.net.HostId.hostId;
 /**
 /**
  * Test for the host identifier.
  * Test for the host identifier.
  */
  */
-public class HostIdTest extends ElementIdTest {
+public class HostIdTest {
 
 
     private static final MacAddress MAC1 = MacAddress.valueOf("00:11:00:00:00:01");
     private static final MacAddress MAC1 = MacAddress.valueOf("00:11:00:00:00:01");
     private static final MacAddress MAC2 = MacAddress.valueOf("00:22:00:00:00:02");
     private static final MacAddress MAC2 = MacAddress.valueOf("00:22:00:00:00:02");
     private static final VlanId VLAN1 = VlanId.vlanId((short) 11);
     private static final VlanId VLAN1 = VlanId.vlanId((short) 11);
     private static final VlanId VLAN2 = VlanId.vlanId((short) 22);
     private static final VlanId VLAN2 = VlanId.vlanId((short) 22);
 
 
-    @Override
     @Test
     @Test
     public void basics() {
     public void basics() {
         new EqualsTester()
         new EqualsTester()
-                .addEqualityGroup(hostId("nic:00:11:00:00:00:01-11"),
-                                  hostId(MAC1, VLAN1))
-                .addEqualityGroup(hostId(MAC2, VLAN2))
+                .addEqualityGroup(hostId(MAC1, VLAN1), hostId(MAC1, VLAN1))
+                .addEqualityGroup(hostId(MAC2, VLAN2), hostId(MAC2, VLAN2))
                 .testEquals();
                 .testEquals();
     }
     }
 
 

@@ -31,7 +31,7 @@ public final class NetTestTools {
 
 
     // Short-hand for producing a host id from a string
     // Short-hand for producing a host id from a string
     public static HostId hid(String id) {
     public static HostId hid(String id) {
-        return hostId("nic:" + id);
+        return hostId(id);
     }
     }
 
 
     // Crates a new device with the specified id
     // Crates a new device with the specified id

@@ -10,9 +10,8 @@ import com.google.common.testing.EqualsTester;
 /**
 /**
  * Test of the port number.
  * Test of the port number.
  */
  */
-public class PortNumberTest extends ElementIdTest {
+public class PortNumberTest {
 
 
-    @Override
     @Test
     @Test
     public void basics() {
     public void basics() {
         new EqualsTester()
         new EqualsTester()

 package org.onlab.onos.net.host;
 package org.onlab.onos.net.host;
 
 
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-import java.util.Set;
-
 import org.junit.Test;
 import org.junit.Test;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.HostLocation;
 import org.onlab.onos.net.HostLocation;
@@ -13,7 +8,8 @@ import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.onlab.packet.VlanId;
 
 
-import com.google.common.collect.Sets;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
 
 
 /**
 /**
  * Test for the default host description.
  * Test for the default host description.
@@ -22,24 +18,22 @@ public class DefualtHostDecriptionTest {
 
 
     private static final MacAddress MAC = MacAddress.valueOf("00:00:11:00:00:01");
     private static final MacAddress MAC = MacAddress.valueOf("00:00:11:00:00:01");
     private static final VlanId VLAN = VlanId.vlanId((short) 10);
     private static final VlanId VLAN = VlanId.vlanId((short) 10);
+    private static final IpPrefix IP = IpPrefix.valueOf("10.0.0.1");
+
     private static final HostLocation LOC = new HostLocation(
     private static final HostLocation LOC = new HostLocation(
-                DeviceId.deviceId("of:foo"),
-                PortNumber.portNumber(100),
-                123L
-            );
-    private static final Set<IpPrefix> IPS = Sets.newHashSet(
-                IpPrefix.valueOf("10.0.0.1"),
-                IpPrefix.valueOf("10.0.0.2")
-            );
+            DeviceId.deviceId("of:foo"),
+            PortNumber.portNumber(100),
+            123L
+    );
 
 
     @Test
     @Test
     public void basics() {
     public void basics() {
         HostDescription host =
         HostDescription host =
-                new DefaultHostDescription(MAC, VLAN, LOC, IPS);
+                new DefaultHostDescription(MAC, VLAN, LOC, IP);
         assertEquals("incorrect mac", MAC, host.hwAddress());
         assertEquals("incorrect mac", MAC, host.hwAddress());
         assertEquals("incorrect vlan", VLAN, host.vlan());
         assertEquals("incorrect vlan", VLAN, host.vlan());
         assertEquals("incorrect location", LOC, host.location());
         assertEquals("incorrect location", LOC, host.location());
-        assertTrue("incorrect ip's", IPS.equals(host.ipAddresses()));
+        assertEquals("incorrect ip's", IP, host.ipAddress());
         assertTrue("incorrect toString", host.toString().contains("vlan=10"));
         assertTrue("incorrect toString", host.toString().contains("vlan=10"));
     }
     }
 
 

 package org.onlab.onos.net.intent;
 package org.onlab.onos.net.intent;
 
 
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+import static org.onlab.onos.net.intent.IntentEvent.Type.FAILED;
+import static org.onlab.onos.net.intent.IntentEvent.Type.INSTALLED;
+import static org.onlab.onos.net.intent.IntentEvent.Type.SUBMITTED;
+import static org.onlab.onos.net.intent.IntentEvent.Type.WITHDRAWN;
 
 
 import java.util.ArrayList;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Collections;
 import java.util.Iterator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.List;
+import java.util.concurrent.Future;
 
 
-import static org.junit.Assert.*;
-import static org.onlab.onos.net.intent.IntentEvent.Type.*;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.onlab.onos.net.flow.CompletedBatchOperation;
 
 
 /**
 /**
  * Suite of tests for the intent service contract.
  * Suite of tests for the intent service contract.
@@ -290,17 +298,19 @@ public class IntentServiceTest {
         }
         }
 
 
         @Override
         @Override
-        public void install(TestInstallableIntent intent) {
+        public Future<CompletedBatchOperation> install(TestInstallableIntent intent) {
             if (fail) {
             if (fail) {
                 throw new IntentException("install failed by design");
                 throw new IntentException("install failed by design");
             }
             }
+            return null;
         }
         }
 
 
         @Override
         @Override
-        public void uninstall(TestInstallableIntent intent) {
+        public Future<CompletedBatchOperation> uninstall(TestInstallableIntent intent) {
             if (fail) {
             if (fail) {
                 throw new IntentException("remove failed by design");
                 throw new IntentException("remove failed by design");
             }
             }
+            return null;
         }
         }
     }
     }
 
 

-package org.onlab.onos.cluster.impl;
+package org.onlab.onos.impl;
 
 
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.ApplicationId;
 import org.onlab.onos.CoreService;
 import org.onlab.onos.CoreService;
 import org.onlab.onos.Version;
 import org.onlab.onos.Version;
 import org.onlab.util.Tools;
 import org.onlab.util.Tools;
 
 
 import java.io.File;
 import java.io.File;
 import java.util.List;
 import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicInteger;
 
 
 /**
 /**
  * Core service implementation.
  * Core service implementation.
@@ -17,9 +21,13 @@ import java.util.List;
 @Service
 @Service
 public class CoreManager implements CoreService {
 public class CoreManager implements CoreService {
 
 
+    private static final AtomicInteger ID_DISPENSER = new AtomicInteger(1);
+
     private static final File VERSION_FILE = new File("../VERSION");
     private static final File VERSION_FILE = new File("../VERSION");
     private static Version version = Version.version("1.0.0-SNAPSHOT");
     private static Version version = Version.version("1.0.0-SNAPSHOT");
 
 
+    private final Map<Short, DefaultApplicationId> appIds = new ConcurrentHashMap<>();
+
     // TODO: work in progress
     // TODO: work in progress
 
 
     @Activate
     @Activate
@@ -35,4 +43,17 @@ public class CoreManager implements CoreService {
         return version;
         return version;
     }
     }
 
 
+    @Override
+    public ApplicationId getAppId(Short id) {
+        return appIds.get(id);
+    }
+
+    @Override
+    public ApplicationId registerApplication(String name) {
+        short id = (short) ID_DISPENSER.getAndIncrement();
+        DefaultApplicationId appId = new DefaultApplicationId(id, name);
+        appIds.put(id, appId);
+        return appId;
+    }
+
 }
 }

+package org.onlab.onos.impl;
+
+import org.onlab.onos.ApplicationId;
+
+import java.util.Objects;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+
+/**
+ * Application id generator class.
+ */
+public class DefaultApplicationId implements ApplicationId {
+
+    private final short id;
+    private final String name;
+
+    // Ban public construction
+    protected DefaultApplicationId(Short id, String identifier) {
+        this.id = id;
+        this.name = identifier;
+    }
+
+    @Override
+    public short id() {
+        return id;
+    }
+
+    @Override
+    public String name() {
+        return name;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(id);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof DefaultApplicationId) {
+            DefaultApplicationId other = (DefaultApplicationId) obj;
+            return Objects.equals(this.id, other.id);
+        }
+        return false;
+    }
+
+    @Override
+    public String toString() {
+        return toStringHelper(this).add("id", id).add("name", name).toString();
+    }
+
+}

+/**
+ *
+ */
+package org.onlab.onos.impl;
\ No newline at end of file

@@ -5,10 +5,12 @@ import static org.slf4j.LoggerFactory.getLogger;
 
 
 import java.util.Iterator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.List;
+import java.util.concurrent.CancellationException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutionException;
 import java.util.concurrent.Future;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeoutException;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
 
 
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
@@ -26,6 +28,7 @@ import org.onlab.onos.net.flow.CompletedBatchOperation;
 import org.onlab.onos.net.flow.FlowEntry;
 import org.onlab.onos.net.flow.FlowEntry;
 import org.onlab.onos.net.flow.FlowRule;
 import org.onlab.onos.net.flow.FlowRule;
 import org.onlab.onos.net.flow.FlowRuleBatchEntry;
 import org.onlab.onos.net.flow.FlowRuleBatchEntry;
+import org.onlab.onos.net.flow.FlowRuleBatchEntry.FlowRuleOperation;
 import org.onlab.onos.net.flow.FlowRuleBatchOperation;
 import org.onlab.onos.net.flow.FlowRuleBatchOperation;
 import org.onlab.onos.net.flow.FlowRuleEvent;
 import org.onlab.onos.net.flow.FlowRuleEvent;
 import org.onlab.onos.net.flow.FlowRuleListener;
 import org.onlab.onos.net.flow.FlowRuleListener;
@@ -52,6 +55,8 @@ public class FlowRuleManager
         extends AbstractProviderRegistry<FlowRuleProvider, FlowRuleProviderService>
         extends AbstractProviderRegistry<FlowRuleProvider, FlowRuleProviderService>
         implements FlowRuleService, FlowRuleProviderRegistry {
         implements FlowRuleService, FlowRuleProviderRegistry {
 
 
+    enum BatchState { STARTED, FINISHED, CANCELLED };
+
     public static final String FLOW_RULE_NULL = "FlowRule cannot be null";
     public static final String FLOW_RULE_NULL = "FlowRule cannot be null";
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
 
 
@@ -144,7 +149,7 @@ public class FlowRuleManager
             FlowRuleBatchOperation batch) {
             FlowRuleBatchOperation batch) {
         Multimap<FlowRuleProvider, FlowRuleBatchEntry> batches =
         Multimap<FlowRuleProvider, FlowRuleBatchEntry> batches =
                 ArrayListMultimap.create();
                 ArrayListMultimap.create();
-        List<Future<Void>> futures = Lists.newArrayList();
+        List<Future<CompletedBatchOperation>> futures = Lists.newArrayList();
         for (FlowRuleBatchEntry fbe : batch.getOperations()) {
         for (FlowRuleBatchEntry fbe : batch.getOperations()) {
             final FlowRule f = fbe.getTarget();
             final FlowRule f = fbe.getTarget();
             final Device device = deviceService.getDevice(f.deviceId());
             final Device device = deviceService.getDevice(f.deviceId());
@@ -165,10 +170,10 @@ public class FlowRuleManager
         for (FlowRuleProvider provider : batches.keySet()) {
         for (FlowRuleProvider provider : batches.keySet()) {
             FlowRuleBatchOperation b =
             FlowRuleBatchOperation b =
                     new FlowRuleBatchOperation(batches.get(provider));
                     new FlowRuleBatchOperation(batches.get(provider));
-            Future<Void> future = provider.executeBatch(b);
+            Future<CompletedBatchOperation> future = provider.executeBatch(b);
             futures.add(future);
             futures.add(future);
         }
         }
-        return new FlowRuleBatchFuture(futures);
+        return new FlowRuleBatchFuture(futures, batches);
     }
     }
 
 
     @Override
     @Override
@@ -341,59 +346,140 @@ public class FlowRuleManager
     private class FlowRuleBatchFuture
     private class FlowRuleBatchFuture
         implements Future<CompletedBatchOperation> {
         implements Future<CompletedBatchOperation> {
 
 
-        private final List<Future<Void>> futures;
+        private final List<Future<CompletedBatchOperation>> futures;
+        private final Multimap<FlowRuleProvider, FlowRuleBatchEntry> batches;
+        private final AtomicReference<BatchState> state;
+        private CompletedBatchOperation overall;
+
+
 
 
-        public FlowRuleBatchFuture(List<Future<Void>> futures) {
+        public FlowRuleBatchFuture(List<Future<CompletedBatchOperation>> futures,
+                Multimap<FlowRuleProvider, FlowRuleBatchEntry> batches) {
             this.futures = futures;
             this.futures = futures;
+            this.batches = batches;
+            state = new AtomicReference<FlowRuleManager.BatchState>();
+            state.set(BatchState.STARTED);
         }
         }
 
 
         @Override
         @Override
         public boolean cancel(boolean mayInterruptIfRunning) {
         public boolean cancel(boolean mayInterruptIfRunning) {
-            // TODO Auto-generated method stub
-            return false;
+            if (state.get() == BatchState.FINISHED) {
+                return false;
+            }
+            if (!state.compareAndSet(BatchState.STARTED, BatchState.CANCELLED)) {
+                return false;
+            }
+            cleanUpBatch();
+            for (Future<CompletedBatchOperation> f : futures) {
+                f.cancel(mayInterruptIfRunning);
+            }
+            return true;
         }
         }
 
 
         @Override
         @Override
         public boolean isCancelled() {
         public boolean isCancelled() {
-            // TODO Auto-generated method stub
-            return false;
+            return state.get() == BatchState.CANCELLED;
         }
         }
 
 
         @Override
         @Override
         public boolean isDone() {
         public boolean isDone() {
-            boolean isDone = true;
-            for (Future<Void> future : futures) {
-                isDone &= future.isDone();
-            }
-            return isDone;
+            return state.get() == BatchState.FINISHED;
         }
         }
 
 
+
         @Override
         @Override
         public CompletedBatchOperation get() throws InterruptedException,
         public CompletedBatchOperation get() throws InterruptedException,
-        ExecutionException {
-            // TODO Auto-generated method stub
-            for (Future<Void> future : futures) {
-                future.get();
+            ExecutionException {
+
+            if (isDone()) {
+                return overall;
+            }
+
+            boolean success = true;
+            List<FlowEntry> failed = Lists.newLinkedList();
+            CompletedBatchOperation completed;
+            for (Future<CompletedBatchOperation> future : futures) {
+                completed = future.get();
+                success = validateBatchOperation(failed, completed, future);
             }
             }
-            return new CompletedBatchOperation();
+
+            return finalizeBatchOperation(success, failed);
+
         }
         }
 
 
         @Override
         @Override
         public CompletedBatchOperation get(long timeout, TimeUnit unit)
         public CompletedBatchOperation get(long timeout, TimeUnit unit)
                 throws InterruptedException, ExecutionException,
                 throws InterruptedException, ExecutionException,
                 TimeoutException {
                 TimeoutException {
-            // TODO we should decrement the timeout
+
+            if (isDone()) {
+                return overall;
+            }
+            boolean success = true;
+            List<FlowEntry> failed = Lists.newLinkedList();
+            CompletedBatchOperation completed;
             long start = System.nanoTime();
             long start = System.nanoTime();
             long end = start + unit.toNanos(timeout);
             long end = start + unit.toNanos(timeout);
-            for (Future<Void> future : futures) {
+
+            for (Future<CompletedBatchOperation> future : futures) {
                 long now = System.nanoTime();
                 long now = System.nanoTime();
                 long thisTimeout = end - now;
                 long thisTimeout = end - now;
-                future.get(thisTimeout, TimeUnit.NANOSECONDS);
+                completed = future.get(thisTimeout, TimeUnit.NANOSECONDS);
+                success = validateBatchOperation(failed, completed, future);
             }
             }
-            return new CompletedBatchOperation();
+            return finalizeBatchOperation(success, failed);
         }
         }
 
 
+        private boolean validateBatchOperation(List<FlowEntry> failed,
+                CompletedBatchOperation completed,
+                Future<CompletedBatchOperation> future) {
+
+            if (isCancelled()) {
+                throw new CancellationException();
+            }
+            if (!completed.isSuccess()) {
+                failed.addAll(completed.failedItems());
+                cleanUpBatch();
+                cancelAllSubBatches();
+                return false;
+            }
+            return true;
+        }
+
+        private void cancelAllSubBatches() {
+            for (Future<CompletedBatchOperation> f : futures) {
+                f.cancel(true);
+            }
+        }
+
+        private CompletedBatchOperation finalizeBatchOperation(boolean success,
+                List<FlowEntry> failed) {
+            synchronized (this) {
+                if (!state.compareAndSet(BatchState.STARTED, BatchState.FINISHED)) {
+                    if (state.get() == BatchState.FINISHED) {
+                        return overall;
+                    }
+                    throw new CancellationException();
+                }
+                overall = new CompletedBatchOperation(success, failed);
+                return overall;
+            }
+        }
+
+        private void cleanUpBatch() {
+            for (FlowRuleBatchEntry fbe : batches.values()) {
+                if (fbe.getOperator() == FlowRuleOperation.ADD ||
+                        fbe.getOperator() == FlowRuleOperation.MODIFY) {
+                    store.deleteFlowRule(fbe.getTarget());
+                } else if (fbe.getOperator() == FlowRuleOperation.REMOVE) {
+                    store.storeFlowRule(fbe.getTarget());
+                }
+            }
+
+        }
     }
     }
 
 
 
 
+
+
 }
 }

@@ -168,7 +168,6 @@ public class HostManager
         checkNotNull(hostId, HOST_ID_NULL);
         checkNotNull(hostId, HOST_ID_NULL);
         HostEvent event = store.removeHost(hostId);
         HostEvent event = store.removeHost(hostId);
         if (event != null) {
         if (event != null) {
-            log.info("Host {} administratively removed", hostId);
             post(event);
             post(event);
         }
         }
     }
     }
@@ -214,7 +213,6 @@ public class HostManager
             HostEvent event = store.createOrUpdateHost(provider().id(), hostId,
             HostEvent event = store.createOrUpdateHost(provider().id(), hostId,
                                                        hostDescription);
                                                        hostDescription);
             if (event != null) {
             if (event != null) {
-                log.debug("Host {} detected", hostId);
                 post(event);
                 post(event);
             }
             }
         }
         }
@@ -225,7 +223,6 @@ public class HostManager
             checkValidity();
             checkValidity();
             HostEvent event = store.removeHost(hostId);
             HostEvent event = store.removeHost(hostId);
             if (event != null) {
             if (event != null) {
-                log.debug("Host {} vanished", hostId);
                 post(event);
                 post(event);
             }
             }
         }
         }

@@ -13,12 +13,17 @@ import static org.onlab.util.Tools.namedThreads;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 
 
 import java.util.ArrayList;
 import java.util.ArrayList;
+import java.util.Iterator;
 import java.util.List;
 import java.util.List;
 import java.util.Map;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutionException;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
@@ -28,6 +33,7 @@ import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.onos.event.AbstractListenerRegistry;
 import org.onlab.onos.event.AbstractListenerRegistry;
 import org.onlab.onos.event.EventDeliveryService;
 import org.onlab.onos.event.EventDeliveryService;
+import org.onlab.onos.net.flow.CompletedBatchOperation;
 import org.onlab.onos.net.intent.InstallableIntent;
 import org.onlab.onos.net.intent.InstallableIntent;
 import org.onlab.onos.net.intent.Intent;
 import org.onlab.onos.net.intent.Intent;
 import org.onlab.onos.net.intent.IntentCompiler;
 import org.onlab.onos.net.intent.IntentCompiler;
@@ -44,7 +50,9 @@ import org.onlab.onos.net.intent.IntentStore;
 import org.onlab.onos.net.intent.IntentStoreDelegate;
 import org.onlab.onos.net.intent.IntentStoreDelegate;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
+import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
 
 
 /**
 /**
  * An implementation of Intent Manager.
  * An implementation of Intent Manager.
@@ -67,7 +75,8 @@ public class IntentManager
     private final AbstractListenerRegistry<IntentEvent, IntentListener>
     private final AbstractListenerRegistry<IntentEvent, IntentListener>
             listenerRegistry = new AbstractListenerRegistry<>();
             listenerRegistry = new AbstractListenerRegistry<>();
 
 
-    private final ExecutorService executor = newSingleThreadExecutor(namedThreads("onos-intents"));
+    private ExecutorService executor;
+    private ExecutorService monitorExecutor;
 
 
     private final IntentStoreDelegate delegate = new InternalStoreDelegate();
     private final IntentStoreDelegate delegate = new InternalStoreDelegate();
     private final TopologyChangeDelegate topoDelegate = new InternalTopoChangeDelegate();
     private final TopologyChangeDelegate topoDelegate = new InternalTopoChangeDelegate();
@@ -86,6 +95,8 @@ public class IntentManager
         store.setDelegate(delegate);
         store.setDelegate(delegate);
         trackerService.setDelegate(topoDelegate);
         trackerService.setDelegate(topoDelegate);
         eventDispatcher.addSink(IntentEvent.class, listenerRegistry);
         eventDispatcher.addSink(IntentEvent.class, listenerRegistry);
+        executor = newSingleThreadExecutor(namedThreads("onos-intents"));
+        monitorExecutor = newSingleThreadExecutor(namedThreads("onos-intent-monitor"));
         log.info("Started");
         log.info("Started");
     }
     }
 
 
@@ -94,6 +105,8 @@ public class IntentManager
         store.unsetDelegate(delegate);
         store.unsetDelegate(delegate);
         trackerService.unsetDelegate(topoDelegate);
         trackerService.unsetDelegate(topoDelegate);
         eventDispatcher.removeSink(IntentEvent.class);
         eventDispatcher.removeSink(IntentEvent.class);
+        executor.shutdown();
+        monitorExecutor.shutdown();
         log.info("Stopped");
         log.info("Stopped");
     }
     }
 
 
@@ -240,14 +253,23 @@ public class IntentManager
         }
         }
     }
     }
 
 
-    // FIXME: To make SDN-IP workable ASAP, only single level compilation is implemented
-    // TODO: implement compilation traversing tree structure
+    /**
+     * Compiles an intent recursively.
+     *
+     * @param intent intent
+     * @return result of compilation
+     */
     private List<InstallableIntent> compileIntent(Intent intent) {
     private List<InstallableIntent> compileIntent(Intent intent) {
+        if (intent instanceof InstallableIntent) {
+            return ImmutableList.of((InstallableIntent) intent);
+        }
+
         List<InstallableIntent> installable = new ArrayList<>();
         List<InstallableIntent> installable = new ArrayList<>();
+        // TODO do we need to registerSubclassCompiler?
         for (Intent compiled : getCompiler(intent).compile(intent)) {
         for (Intent compiled : getCompiler(intent).compile(intent)) {
-            InstallableIntent installableIntent = (InstallableIntent) compiled;
-            installable.add(installableIntent);
+            installable.addAll(compileIntent(compiled));
         }
         }
+
         return installable;
         return installable;
     }
     }
 
 
@@ -261,6 +283,7 @@ public class IntentManager
         // Indicate that the intent is entering the installing phase.
         // Indicate that the intent is entering the installing phase.
         store.setState(intent, INSTALLING);
         store.setState(intent, INSTALLING);
 
 
+        List<Future<CompletedBatchOperation>> installFutures = Lists.newArrayList();
         try {
         try {
             List<InstallableIntent> installables = store.getInstallableIntents(intent.id());
             List<InstallableIntent> installables = store.getInstallableIntents(intent.id());
             if (installables != null) {
             if (installables != null) {
@@ -268,17 +291,20 @@ public class IntentManager
                     registerSubclassInstallerIfNeeded(installable);
                     registerSubclassInstallerIfNeeded(installable);
                     trackerService.addTrackedResources(intent.id(),
                     trackerService.addTrackedResources(intent.id(),
                                                        installable.requiredLinks());
                                                        installable.requiredLinks());
-                    getInstaller(installable).install(installable);
+                    Future<CompletedBatchOperation> future = getInstaller(installable).install(installable);
+                    installFutures.add(future);
                 }
                 }
             }
             }
-            eventDispatcher.post(store.setState(intent, INSTALLED));
-
+            // FIXME we have to wait for the installable intents
+            //eventDispatcher.post(store.setState(intent, INSTALLED));
+            monitorExecutor.execute(new IntentInstallMonitor(intent, installFutures, INSTALLED));
         } catch (Exception e) {
         } catch (Exception e) {
             log.warn("Unable to install intent {} due to: {}", intent.id(), e);
             log.warn("Unable to install intent {} due to: {}", intent.id(), e);
-            uninstallIntent(intent);
+            uninstallIntent(intent, RECOMPILING);
 
 
             // If compilation failed, kick off the recompiling phase.
             // If compilation failed, kick off the recompiling phase.
-            executeRecompilingPhase(intent);
+            // FIXME
+            //executeRecompilingPhase(intent);
         }
         }
     }
     }
 
 
@@ -327,12 +353,14 @@ public class IntentManager
     private void executeWithdrawingPhase(Intent intent) {
     private void executeWithdrawingPhase(Intent intent) {
         // Indicate that the intent is being withdrawn.
         // Indicate that the intent is being withdrawn.
         store.setState(intent, WITHDRAWING);
         store.setState(intent, WITHDRAWING);
-        uninstallIntent(intent);
+        uninstallIntent(intent, WITHDRAWN);
 
 
         // If all went well, disassociate the top-level intent with its
         // If all went well, disassociate the top-level intent with its
         // installable derivatives and mark it as withdrawn.
         // installable derivatives and mark it as withdrawn.
-        store.removeInstalledIntents(intent.id());
-        eventDispatcher.post(store.setState(intent, WITHDRAWN));
+        // FIXME need to clean up
+        //store.removeInstalledIntents(intent.id());
+        // FIXME
+        //eventDispatcher.post(store.setState(intent, WITHDRAWN));
     }
     }
 
 
     /**
     /**
@@ -340,14 +368,17 @@ public class IntentManager
      *
      *
      * @param intent intent to be uninstalled
      * @param intent intent to be uninstalled
      */
      */
-    private void uninstallIntent(Intent intent) {
+    private void uninstallIntent(Intent intent, IntentState nextState) {
+        List<Future<CompletedBatchOperation>> uninstallFutures = Lists.newArrayList();
         try {
         try {
             List<InstallableIntent> installables = store.getInstallableIntents(intent.id());
             List<InstallableIntent> installables = store.getInstallableIntents(intent.id());
             if (installables != null) {
             if (installables != null) {
                 for (InstallableIntent installable : installables) {
                 for (InstallableIntent installable : installables) {
-                    getInstaller(installable).uninstall(installable);
+                    Future<CompletedBatchOperation> future = getInstaller(installable).uninstall(installable);
+                    uninstallFutures.add(future);
                 }
                 }
             }
             }
+            monitorExecutor.execute(new IntentInstallMonitor(intent, uninstallFutures, nextState));
         } catch (IntentException e) {
         } catch (IntentException e) {
             log.warn("Unable to uninstall intent {} due to: {}", intent.id(), e);
             log.warn("Unable to uninstall intent {} due to: {}", intent.id(), e);
         }
         }
@@ -422,9 +453,10 @@ public class IntentManager
             // Attempt recompilation of the specified intents first.
             // Attempt recompilation of the specified intents first.
             for (IntentId intentId : intentIds) {
             for (IntentId intentId : intentIds) {
                 Intent intent = getIntent(intentId);
                 Intent intent = getIntent(intentId);
-                uninstallIntent(intent);
+                uninstallIntent(intent, RECOMPILING);
 
 
-                executeRecompilingPhase(intent);
+                //FIXME
+                //executeRecompilingPhase(intent);
             }
             }
 
 
             if (compileAllFailed) {
             if (compileAllFailed) {
@@ -460,4 +492,49 @@ public class IntentManager
         }
         }
     }
     }
 
 
+    private class IntentInstallMonitor implements Runnable {
+
+        private final Intent intent;
+        private final List<Future<CompletedBatchOperation>> futures;
+        private final IntentState nextState;
+
+        public IntentInstallMonitor(Intent intent,
+                List<Future<CompletedBatchOperation>> futures, IntentState nextState) {
+            this.intent = intent;
+            this.futures = futures;
+            this.nextState = nextState;
+        }
+
+        private void updateIntent(Intent intent) {
+            if (nextState == RECOMPILING) {
+                executor.execute(new IntentTask(nextState, intent));
+            } else if (nextState == INSTALLED || nextState == WITHDRAWN) {
+                eventDispatcher.post(store.setState(intent, nextState));
+            } else {
+                log.warn("Invalid next intent state {} for intent {}", nextState, intent);
+            }
+        }
+
+        @Override
+        public void run() {
+            for (Iterator<Future<CompletedBatchOperation>> i = futures.iterator(); i.hasNext();) {
+                Future<CompletedBatchOperation> future = i.next();
+                try {
+                    // TODO: we may want to get the future here and go back to the future.
+                    CompletedBatchOperation completed = future.get(100, TimeUnit.NANOSECONDS);
+                    // TODO check if future succeeded and if not report fail items
+                    i.remove();
+
+                } catch (TimeoutException | InterruptedException | ExecutionException te) {
+                    log.debug("Intallations of intent {} is still pending", intent);
+                }
+            }
+            if (futures.isEmpty()) {
+                updateIntent(intent);
+            } else {
+                // resubmit ourselves if we are not done yet
+                monitorExecutor.submit(this);
+            }
+        }
+    }
 }
 }

@@ -5,7 +5,7 @@ import static org.slf4j.LoggerFactory.getLogger;
 
 
 import java.util.Iterator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.List;
-import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
 
 
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
@@ -13,8 +13,10 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.ApplicationId;
+import org.onlab.onos.CoreService;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.Link;
 import org.onlab.onos.net.Link;
+import org.onlab.onos.net.flow.CompletedBatchOperation;
 import org.onlab.onos.net.flow.DefaultFlowRule;
 import org.onlab.onos.net.flow.DefaultFlowRule;
 import org.onlab.onos.net.flow.DefaultTrafficSelector;
 import org.onlab.onos.net.flow.DefaultTrafficSelector;
 import org.onlab.onos.net.flow.FlowRule;
 import org.onlab.onos.net.flow.FlowRule;
@@ -45,10 +47,14 @@ public class PathIntentInstaller implements IntentInstaller<PathIntent> {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected FlowRuleService flowRuleService;
     protected FlowRuleService flowRuleService;
 
 
-    private final ApplicationId appId = ApplicationId.getAppId();
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected CoreService coreService;
+
+    private ApplicationId appId;
 
 
     @Activate
     @Activate
     public void activate() {
     public void activate() {
+        appId = coreService.registerApplication("org.onlab.onos.net.intent");
         intentManager.registerInstaller(PathIntent.class, this);
         intentManager.registerInstaller(PathIntent.class, this);
     }
     }
 
 
@@ -57,8 +63,26 @@ public class PathIntentInstaller implements IntentInstaller<PathIntent> {
         intentManager.unregisterInstaller(PathIntent.class);
         intentManager.unregisterInstaller(PathIntent.class);
     }
     }
 
 
+    /**
+     * Apply a list of FlowRules.
+     *
+     * @param rules rules to apply
+     */
+    private Future<CompletedBatchOperation> applyBatch(List<FlowRuleBatchEntry> rules) {
+        FlowRuleBatchOperation batch = new FlowRuleBatchOperation(rules);
+        Future<CompletedBatchOperation> future = flowRuleService.applyBatch(batch);
+        return future;
+//        try {
+//            //FIXME don't do this here
+//            future.get();
+//        } catch (InterruptedException | ExecutionException e) {
+//            // TODO Auto-generated catch block
+//            e.printStackTrace();
+//        }
+    }
+
     @Override
     @Override
-    public void install(PathIntent intent) {
+    public Future<CompletedBatchOperation> install(PathIntent intent) {
         TrafficSelector.Builder builder =
         TrafficSelector.Builder builder =
                 DefaultTrafficSelector.builder(intent.selector());
                 DefaultTrafficSelector.builder(intent.selector());
         Iterator<Link> links = intent.path().links().iterator();
         Iterator<Link> links = intent.path().links().iterator();
@@ -74,20 +98,14 @@ public class PathIntentInstaller implements IntentInstaller<PathIntent> {
                     builder.build(), treatment,
                     builder.build(), treatment,
                     123, appId, 600);
                     123, appId, 600);
             rules.add(new FlowRuleBatchEntry(FlowRuleOperation.ADD, rule));
             rules.add(new FlowRuleBatchEntry(FlowRuleOperation.ADD, rule));
-            //flowRuleService.applyFlowRules(rule);
             prev = link.dst();
             prev = link.dst();
         }
         }
-        FlowRuleBatchOperation batch = new FlowRuleBatchOperation(rules);
-        try {
-            flowRuleService.applyBatch(batch).get();
-        } catch (InterruptedException | ExecutionException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
-        }
+
+        return applyBatch(rules);
     }
     }
 
 
     @Override
     @Override
-    public void uninstall(PathIntent intent) {
+    public Future<CompletedBatchOperation> uninstall(PathIntent intent) {
         TrafficSelector.Builder builder =
         TrafficSelector.Builder builder =
                 DefaultTrafficSelector.builder(intent.selector());
                 DefaultTrafficSelector.builder(intent.selector());
         Iterator<Link> links = intent.path().links().iterator();
         Iterator<Link> links = intent.path().links().iterator();
@@ -103,15 +121,131 @@ public class PathIntentInstaller implements IntentInstaller<PathIntent> {
                     builder.build(), treatment,
                     builder.build(), treatment,
                     123, appId, 600);
                     123, appId, 600);
             rules.add(new FlowRuleBatchEntry(FlowRuleOperation.REMOVE, rule));
             rules.add(new FlowRuleBatchEntry(FlowRuleOperation.REMOVE, rule));
-            //flowRuleService.removeFlowRules(rule);
             prev = link.dst();
             prev = link.dst();
         }
         }
-        FlowRuleBatchOperation batch = new FlowRuleBatchOperation(rules);
-        try {
-            flowRuleService.applyBatch(batch).get();
-        } catch (InterruptedException | ExecutionException e) {
-            // TODO Auto-generated catch block
-            e.printStackTrace();
+        return applyBatch(rules);
+    }
+
+    // TODO refactor below this line... ----------------------------
+
+    /**
+     * Generates the series of MatchActionOperations from the
+     * {@link FlowBatchOperation}.
+     * <p>
+     * FIXME: Currently supporting PacketPathFlow and SingleDstTreeFlow only.
+     * <p>
+     * FIXME: MatchActionOperations should have dependency field to the other
+     * match action operations, and this method should use this.
+     *
+     * @param op the {@link FlowBatchOperation} object
+     * @return the list of {@link MatchActionOperations} objects
+     */
+    /*
+    private List<MatchActionOperations>
+            generateMatchActionOperationsList(FlowBatchOperation op) {
+
+        // MatchAction operations at head (ingress) switches.
+        MatchActionOperations headOps = matchActionService.createOperationsList();
+
+        // MatchAction operations at rest of the switches.
+        MatchActionOperations tailOps = matchActionService.createOperationsList();
+
+        MatchActionOperations removeOps = matchActionService.createOperationsList();
+
+        for (BatchOperationEntry<Operator, ?> e : op.getOperations()) {
+
+            if (e.getOperator() == FlowBatchOperation.Operator.ADD) {
+                generateInstallMatchActionOperations(e, tailOps, headOps);
+            } else if (e.getOperator() == FlowBatchOperation.Operator.REMOVE) {
+                generateRemoveMatchActionOperations(e, removeOps);
+            } else {
+                throw new UnsupportedOperationException(
+                        "FlowManager supports ADD and REMOVE operations only.");
+            }
+
+        }
+
+        return Arrays.asList(tailOps, headOps, removeOps);
+    }
+    */
+
+    /**
+     * Generates MatchActionOperations for an INSTALL FlowBatchOperation.
+     * <p/>
+     * FIXME: Currently only supports flows that generate exactly two match
+     * action operation sets.
+     *
+     * @param e Flow BatchOperationEntry
+     * @param tailOps MatchActionOperation set that the tail
+     * MatchActionOperations will be placed in
+     * @param headOps MatchActionOperation set that the head
+     * MatchActionOperations will be placed in
+     */
+    /*
+    private void generateInstallMatchActionOperations(
+            BatchOperationEntry<Operator, ?> e,
+            MatchActionOperations tailOps,
+            MatchActionOperations headOps) {
+
+        if (!(e.getTarget() instanceof Flow)) {
+            throw new IllegalStateException(
+                    "The target is not Flow object: " + e.getTarget());
+        }
+
+        // Compile flows to match-actions
+        Flow flow = (Flow) e.getTarget();
+        List<MatchActionOperations> maOps = flow.compile(
+                e.getOperator(), matchActionService);
+        verifyNotNull(maOps, "Could not compile the flow: " + flow);
+        verify(maOps.size() == 2,
+                "The flow generates unspported match-action operations.");
+
+        // Map FlowId to MatchActionIds
+        for (MatchActionOperations maOp : maOps) {
+            for (MatchActionOperationEntry entry : maOp.getOperations()) {
+                flowMatchActionsMap.put(
+                        KryoFactory.serialize(flow.getId()),
+                        KryoFactory.serialize(entry.getTarget()));
+            }
+        }
+
+        // Merge match-action operations
+        for (MatchActionOperationEntry mae : maOps.get(0).getOperations()) {
+            verify(mae.getOperator() == MatchActionOperations.Operator.INSTALL);
+            tailOps.addOperation(mae);
+        }
+        for (MatchActionOperationEntry mae : maOps.get(1).getOperations()) {
+            verify(mae.getOperator() == MatchActionOperations.Operator.INSTALL);
+            headOps.addOperation(mae);
+        }
+    }
+    */
+    /**
+     * Generates MatchActionOperations for a REMOVE FlowBatchOperation.
+     *
+     * @param e Flow BatchOperationEntry
+     * @param removeOps MatchActionOperation set that the remove
+     * MatchActionOperations will be placed in
+     */
+    /*
+    private void generateRemoveMatchActionOperations(
+            BatchOperationEntry<Operator, ?> e,
+            MatchActionOperations removeOps) {
+
+        if (!(e.getTarget() instanceof FlowId)) {
+            throw new IllegalStateException(
+                    "The target is not a FlowId object: " + e.getTarget());
+        }
+
+        // Compile flows to match-actions
+        FlowId flowId = (FlowId) e.getTarget();
+
+        for (byte[] matchActionIdBytes :
+            flowMatchActionsMap.remove(KryoFactory.serialize(flowId))) {
+            MatchActionId matchActionId = KryoFactory.deserialize(matchActionIdBytes);
+            removeOps.addOperation(new MatchActionOperationEntry(
+                    MatchActionOperations.Operator.REMOVE, matchActionId));
         }
         }
     }
     }
+    */
 }
 }

@@ -55,6 +55,7 @@ public class ProxyArpManager implements ProxyArpService {
     private static final String REQUEST_NULL = "Arp request cannot be null.";
     private static final String REQUEST_NULL = "Arp request cannot be null.";
     private static final String REQUEST_NOT_ARP = "Ethernet frame does not contain ARP request.";
     private static final String REQUEST_NOT_ARP = "Ethernet frame does not contain ARP request.";
     private static final String NOT_ARP_REQUEST = "ARP is not a request.";
     private static final String NOT_ARP_REQUEST = "ARP is not a request.";
+    private static final String NOT_ARP_REPLY = "ARP is not a reply.";
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected HostService hostService;
     protected HostService hostService;
@@ -141,7 +142,7 @@ public class ProxyArpManager implements ProxyArpService {
         checkArgument(eth.getEtherType() == Ethernet.TYPE_ARP,
         checkArgument(eth.getEtherType() == Ethernet.TYPE_ARP,
                 REQUEST_NOT_ARP);
                 REQUEST_NOT_ARP);
         ARP arp = (ARP) eth.getPayload();
         ARP arp = (ARP) eth.getPayload();
-        checkArgument(arp.getOpCode() == ARP.OP_REPLY, NOT_ARP_REQUEST);
+        checkArgument(arp.getOpCode() == ARP.OP_REPLY, NOT_ARP_REPLY);
 
 
         Host h = hostService.getHost(HostId.hostId(eth.getDestinationMAC(),
         Host h = hostService.getHost(HostId.hostId(eth.getDestinationMAC(),
                 VlanId.vlanId(eth.getVlanID())));
                 VlanId.vlanId(eth.getVlanID())));

@@ -22,9 +22,9 @@ import org.onlab.onos.net.Link;
 import org.onlab.onos.net.Path;
 import org.onlab.onos.net.Path;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.host.HostService;
 import org.onlab.onos.net.host.HostService;
-import org.onlab.onos.net.topology.PathService;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.topology.LinkWeight;
 import org.onlab.onos.net.topology.LinkWeight;
+import org.onlab.onos.net.topology.PathService;
 import org.onlab.onos.net.topology.Topology;
 import org.onlab.onos.net.topology.Topology;
 import org.onlab.onos.net.topology.TopologyService;
 import org.onlab.onos.net.topology.TopologyService;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
@@ -33,7 +33,6 @@ import java.util.List;
 import java.util.Set;
 import java.util.Set;
 
 
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Preconditions.checkNotNull;
-import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 
 
 /**
 /**
@@ -162,8 +161,8 @@ public class PathManager implements PathService {
     // edge link since the src or dst are really an infrastructure device.
     // edge link since the src or dst are really an infrastructure device.
     private static class NotHost extends DefaultEdgeLink implements EdgeLink {
     private static class NotHost extends DefaultEdgeLink implements EdgeLink {
         NotHost() {
         NotHost() {
-            super(PID, new ConnectPoint(HostId.hostId("nic:none"), P0),
-                  new HostLocation(deviceId("none:none"), P0, 0L), false);
+            super(PID, new ConnectPoint(HostId.NONE, P0),
+                  new HostLocation(DeviceId.NONE, P0, 0L), false);
         }
         }
     }
     }
 }
 }

@@ -19,6 +19,7 @@ import org.junit.Before;
 import org.junit.Test;
 import org.junit.Test;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.event.impl.TestEventDispatcher;
 import org.onlab.onos.event.impl.TestEventDispatcher;
+import org.onlab.onos.impl.DefaultApplicationId;
 import org.onlab.onos.net.DefaultDevice;
 import org.onlab.onos.net.DefaultDevice;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.Device.Type;
 import org.onlab.onos.net.Device.Type;
@@ -28,6 +29,7 @@ import org.onlab.onos.net.Port;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.device.DeviceListener;
 import org.onlab.onos.net.device.DeviceListener;
 import org.onlab.onos.net.device.DeviceService;
 import org.onlab.onos.net.device.DeviceService;
+import org.onlab.onos.net.flow.CompletedBatchOperation;
 import org.onlab.onos.net.flow.DefaultFlowEntry;
 import org.onlab.onos.net.flow.DefaultFlowEntry;
 import org.onlab.onos.net.flow.DefaultFlowRule;
 import org.onlab.onos.net.flow.DefaultFlowRule;
 import org.onlab.onos.net.flow.FlowEntry;
 import org.onlab.onos.net.flow.FlowEntry;
@@ -58,6 +60,8 @@ import com.google.common.collect.Sets;
  */
  */
 public class FlowRuleManagerTest {
 public class FlowRuleManagerTest {
 
 
+
+
     private static final ProviderId PID = new ProviderId("of", "foo");
     private static final ProviderId PID = new ProviderId("of", "foo");
     private static final DeviceId DID = DeviceId.deviceId("of:001");
     private static final DeviceId DID = DeviceId.deviceId("of:001");
     private static final int TIMEOUT = 10;
     private static final int TIMEOUT = 10;
@@ -86,7 +90,7 @@ public class FlowRuleManagerTest {
         mgr.addListener(listener);
         mgr.addListener(listener);
         provider = new TestProvider(PID);
         provider = new TestProvider(PID);
         providerService = registry.register(provider);
         providerService = registry.register(provider);
-        appId = ApplicationId.getAppId();
+        appId = new TestApplicationId((short) 0, "FlowRuleManagerTest");
         assertTrue("provider should be registered",
         assertTrue("provider should be registered",
                 registry.getProviders().contains(provider.id()));
                 registry.getProviders().contains(provider.id()));
     }
     }
@@ -408,7 +412,7 @@ public class FlowRuleManagerTest {
         }
         }
 
 
         @Override
         @Override
-        public Future<Void> executeBatch(
+        public Future<CompletedBatchOperation> executeBatch(
                 BatchOperation<FlowRuleBatchEntry> batch) {
                 BatchOperation<FlowRuleBatchEntry> batch) {
             // TODO Auto-generated method stub
             // TODO Auto-generated method stub
             return null;
             return null;
@@ -474,4 +478,11 @@ public class FlowRuleManagerTest {
 
 
     }
     }
 
 
+    public class TestApplicationId extends DefaultApplicationId {
+
+        public TestApplicationId(short id, String name) {
+            super(id, name);
+        }
+    }
+
 }
 }

@@ -58,8 +58,6 @@ public class HostManagerTest {
 
 
     private static final IpPrefix IP1 = IpPrefix.valueOf("10.0.0.1");
     private static final IpPrefix IP1 = IpPrefix.valueOf("10.0.0.1");
     private static final IpPrefix IP2 = IpPrefix.valueOf("10.0.0.2");
     private static final IpPrefix IP2 = IpPrefix.valueOf("10.0.0.2");
-    private static final Set<IpPrefix> IPSET1 = Sets.newHashSet(IP1);
-    private static final Set<IpPrefix> IPSET2 = Sets.newHashSet(IP2);
 
 
     private static final DeviceId DID1 = DeviceId.deviceId("of:001");
     private static final DeviceId DID1 = DeviceId.deviceId("of:001");
     private static final DeviceId DID2 = DeviceId.deviceId("of:002");
     private static final DeviceId DID2 = DeviceId.deviceId("of:002");
@@ -94,14 +92,14 @@ public class HostManagerTest {
         provider = new TestHostProvider();
         provider = new TestHostProvider();
         providerService = registry.register(provider);
         providerService = registry.register(provider);
         assertTrue("provider should be registered",
         assertTrue("provider should be registered",
-                registry.getProviders().contains(provider.id()));
+                   registry.getProviders().contains(provider.id()));
     }
     }
 
 
     @After
     @After
     public void tearDown() {
     public void tearDown() {
         registry.unregister(provider);
         registry.unregister(provider);
         assertFalse("provider should not be registered",
         assertFalse("provider should not be registered",
-                registry.getProviders().contains(provider.id()));
+                    registry.getProviders().contains(provider.id()));
 
 
         mgr.removeListener(listener);
         mgr.removeListener(listener);
         mgr.deactivate();
         mgr.deactivate();
@@ -109,8 +107,8 @@ public class HostManagerTest {
     }
     }
 
 
     private void detect(HostId hid, MacAddress mac, VlanId vlan,
     private void detect(HostId hid, MacAddress mac, VlanId vlan,
-            HostLocation loc, Set<IpPrefix> ips) {
-        HostDescription descr = new DefaultHostDescription(mac, vlan, loc, ips);
+                        HostLocation loc, IpPrefix ip) {
+        HostDescription descr = new DefaultHostDescription(mac, vlan, loc, ip);
         providerService.hostDetected(hid, descr);
         providerService.hostDetected(hid, descr);
         assertNotNull("host should be found", mgr.getHost(hid));
         assertNotNull("host should be found", mgr.getHost(hid));
     }
     }
@@ -130,26 +128,26 @@ public class HostManagerTest {
         assertNull("host shouldn't be found", mgr.getHost(HID1));
         assertNull("host shouldn't be found", mgr.getHost(HID1));
 
 
         // host addition
         // host addition
-        detect(HID1, MAC1, VLAN1, LOC1, IPSET1);
+        detect(HID1, MAC1, VLAN1, LOC1, IP1);
         assertEquals("exactly one should be found", 1, mgr.getHostCount());
         assertEquals("exactly one should be found", 1, mgr.getHostCount());
-        detect(HID2, MAC2, VLAN2, LOC2, IPSET1);
+        detect(HID2, MAC2, VLAN2, LOC2, IP1);
         assertEquals("two hosts should be found", 2, mgr.getHostCount());
         assertEquals("two hosts should be found", 2, mgr.getHostCount());
         validateEvents(HOST_ADDED, HOST_ADDED);
         validateEvents(HOST_ADDED, HOST_ADDED);
 
 
         // host motion
         // host motion
-        detect(HID1, MAC1, VLAN1, LOC2, IPSET1);
+        detect(HID1, MAC1, VLAN1, LOC2, IP1);
         validateEvents(HOST_MOVED);
         validateEvents(HOST_MOVED);
         assertEquals("only two hosts should be found", 2, mgr.getHostCount());
         assertEquals("only two hosts should be found", 2, mgr.getHostCount());
 
 
         // host update
         // host update
-        detect(HID1, MAC1, VLAN1, LOC2, IPSET2);
+        detect(HID1, MAC1, VLAN1, LOC2, IP2);
         validateEvents(HOST_UPDATED);
         validateEvents(HOST_UPDATED);
         assertEquals("only two hosts should be found", 2, mgr.getHostCount());
         assertEquals("only two hosts should be found", 2, mgr.getHostCount());
     }
     }
 
 
     @Test
     @Test
     public void hostVanished() {
     public void hostVanished() {
-        detect(HID1, MAC1, VLAN1, LOC1, IPSET1);
+        detect(HID1, MAC1, VLAN1, LOC1, IP1);
         providerService.hostVanished(HID1);
         providerService.hostVanished(HID1);
         validateEvents(HOST_ADDED, HOST_REMOVED);
         validateEvents(HOST_ADDED, HOST_REMOVED);
 
 
@@ -157,7 +155,7 @@ public class HostManagerTest {
     }
     }
 
 
     private void validateHosts(
     private void validateHosts(
-            String msg, Iterable<Host> hosts, HostId ... ids) {
+            String msg, Iterable<Host> hosts, HostId... ids) {
         Set<HostId> hids = Sets.newHashSet(ids);
         Set<HostId> hids = Sets.newHashSet(ids);
         for (Host h : hosts) {
         for (Host h : hosts) {
             assertTrue(msg, hids.remove(h.id()));
             assertTrue(msg, hids.remove(h.id()));
@@ -167,8 +165,8 @@ public class HostManagerTest {
 
 
     @Test
     @Test
     public void getHosts() {
     public void getHosts() {
-        detect(HID1, MAC1, VLAN1, LOC1, IPSET1);
-        detect(HID2, MAC2, VLAN1, LOC2, IPSET2);
+        detect(HID1, MAC1, VLAN1, LOC1, IP1);
+        detect(HID2, MAC2, VLAN1, LOC2, IP2);
 
 
         validateHosts("host not properly stored", mgr.getHosts(), HID1, HID2);
         validateHosts("host not properly stored", mgr.getHosts(), HID1, HID2);
         validateHosts("can't get hosts by VLAN", mgr.getHostsByVlan(VLAN1), HID1, HID2);
         validateHosts("can't get hosts by VLAN", mgr.getHostsByVlan(VLAN1), HID1, HID2);
@@ -210,7 +208,7 @@ public class HostManagerTest {
     @Test
     @Test
     public void bindAddressesToPort() {
     public void bindAddressesToPort() {
         PortAddresses add1 = new PortAddresses(CP1,
         PortAddresses add1 = new PortAddresses(CP1,
-                Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
+                                               Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
 
 
         mgr.bindAddressesToPort(add1);
         mgr.bindAddressesToPort(add1);
         PortAddresses storedAddresses = mgr.getAddressBindingsForPort(CP1);
         PortAddresses storedAddresses = mgr.getAddressBindingsForPort(CP1);
@@ -241,7 +239,7 @@ public class HostManagerTest {
     @Test
     @Test
     public void unbindAddressesFromPort() {
     public void unbindAddressesFromPort() {
         PortAddresses add1 = new PortAddresses(CP1,
         PortAddresses add1 = new PortAddresses(CP1,
-                Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
+                                               Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
 
 
         mgr.bindAddressesToPort(add1);
         mgr.bindAddressesToPort(add1);
         PortAddresses storedAddresses = mgr.getAddressBindingsForPort(CP1);
         PortAddresses storedAddresses = mgr.getAddressBindingsForPort(CP1);
@@ -250,7 +248,7 @@ public class HostManagerTest {
         assertNotNull(storedAddresses.mac());
         assertNotNull(storedAddresses.mac());
 
 
         PortAddresses rem1 = new PortAddresses(CP1,
         PortAddresses rem1 = new PortAddresses(CP1,
-                Sets.newHashSet(PREFIX1), null);
+                                               Sets.newHashSet(PREFIX1), null);
 
 
         mgr.unbindAddressesFromPort(rem1);
         mgr.unbindAddressesFromPort(rem1);
         storedAddresses = mgr.getAddressBindingsForPort(CP1);
         storedAddresses = mgr.getAddressBindingsForPort(CP1);
@@ -267,7 +265,7 @@ public class HostManagerTest {
         assertNull(storedAddresses.mac());
         assertNull(storedAddresses.mac());
 
 
         PortAddresses rem3 = new PortAddresses(CP1,
         PortAddresses rem3 = new PortAddresses(CP1,
-                Sets.newHashSet(PREFIX2), MAC1);
+                                               Sets.newHashSet(PREFIX2), MAC1);
 
 
         mgr.unbindAddressesFromPort(rem3);
         mgr.unbindAddressesFromPort(rem3);
         storedAddresses = mgr.getAddressBindingsForPort(CP1);
         storedAddresses = mgr.getAddressBindingsForPort(CP1);
@@ -279,7 +277,7 @@ public class HostManagerTest {
     @Test
     @Test
     public void clearAddresses() {
     public void clearAddresses() {
         PortAddresses add1 = new PortAddresses(CP1,
         PortAddresses add1 = new PortAddresses(CP1,
-                Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
+                                               Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
 
 
         mgr.bindAddressesToPort(add1);
         mgr.bindAddressesToPort(add1);
         PortAddresses storedAddresses = mgr.getAddressBindingsForPort(CP1);
         PortAddresses storedAddresses = mgr.getAddressBindingsForPort(CP1);
@@ -297,7 +295,7 @@ public class HostManagerTest {
     @Test
     @Test
     public void getAddressBindingsForPort() {
     public void getAddressBindingsForPort() {
         PortAddresses add1 = new PortAddresses(CP1,
         PortAddresses add1 = new PortAddresses(CP1,
-                Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
+                                               Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
 
 
         mgr.bindAddressesToPort(add1);
         mgr.bindAddressesToPort(add1);
         PortAddresses storedAddresses = mgr.getAddressBindingsForPort(CP1);
         PortAddresses storedAddresses = mgr.getAddressBindingsForPort(CP1);
@@ -314,7 +312,7 @@ public class HostManagerTest {
         assertTrue(storedAddresses.isEmpty());
         assertTrue(storedAddresses.isEmpty());
 
 
         PortAddresses add1 = new PortAddresses(CP1,
         PortAddresses add1 = new PortAddresses(CP1,
-                Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
+                                               Sets.newHashSet(PREFIX1, PREFIX2), MAC1);
 
 
         mgr.bindAddressesToPort(add1);
         mgr.bindAddressesToPort(add1);
 
 
@@ -323,7 +321,7 @@ public class HostManagerTest {
         assertTrue(storedAddresses.size() == 1);
         assertTrue(storedAddresses.size() == 1);
 
 
         PortAddresses add2 = new PortAddresses(CP2,
         PortAddresses add2 = new PortAddresses(CP2,
-                Sets.newHashSet(PREFIX3), MAC2);
+                                               Sets.newHashSet(PREFIX3), MAC2);
 
 
         mgr.bindAddressesToPort(add2);
         mgr.bindAddressesToPort(add2);
 
 

+package org.onlab.onos.net.proxyarp.impl;
+
+import static org.easymock.EasyMock.anyObject;
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.replay;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DefaultHost;
+import org.onlab.onos.net.Device;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.Host;
+import org.onlab.onos.net.HostId;
+import org.onlab.onos.net.HostLocation;
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.Port;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.device.DeviceListener;
+import org.onlab.onos.net.device.DeviceService;
+import org.onlab.onos.net.flow.instructions.Instruction;
+import org.onlab.onos.net.flow.instructions.Instructions.OutputInstruction;
+import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.link.LinkListener;
+import org.onlab.onos.net.link.LinkService;
+import org.onlab.onos.net.packet.OutboundPacket;
+import org.onlab.onos.net.packet.PacketProcessor;
+import org.onlab.onos.net.packet.PacketService;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.packet.ARP;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
+import org.onlab.packet.VlanId;
+
+import com.google.common.collect.Sets;
+
+/**
+ * Tests for the {@link ProxyArpManager} class.
+ */
+public class ProxyArpManagerTest {
+
+    private static final int NUM_DEVICES = 4;
+    private static final int NUM_PORTS_PER_DEVICE = 3;
+    private static final int NUM_FLOOD_PORTS = 4;
+
+    private static final IpPrefix IP1 = IpPrefix.valueOf("10.0.0.1/24");
+    private static final IpPrefix IP2 = IpPrefix.valueOf("10.0.0.2/24");
+
+    private static final ProviderId PID = new ProviderId("of", "foo");
+
+    private static final VlanId VLAN1 = VlanId.vlanId((short) 1);
+    private static final VlanId VLAN2 = VlanId.vlanId((short) 2);
+    private static final MacAddress MAC1 = MacAddress.valueOf("00:00:11:00:00:01");
+    private static final MacAddress MAC2 = MacAddress.valueOf("00:00:22:00:00:02");
+    private static final HostId HID1 = HostId.hostId(MAC1, VLAN1);
+    private static final HostId HID2 = HostId.hostId(MAC2, VLAN1);
+
+    private static final DeviceId DID1 = getDeviceId(1);
+    private static final DeviceId DID2 = getDeviceId(2);
+    private static final PortNumber P1 = PortNumber.portNumber(1);
+    private static final HostLocation LOC1 = new HostLocation(DID1, P1, 123L);
+    private static final HostLocation LOC2 = new HostLocation(DID2, P1, 123L);
+
+    private ProxyArpManager proxyArp;
+
+    private TestPacketService packetService;
+
+    private DeviceService deviceService;
+    private LinkService linkService;
+    private HostService hostService;
+
+    @Before
+    public void setUp() throws Exception {
+        proxyArp = new ProxyArpManager();
+        packetService = new TestPacketService();
+        proxyArp.packetService = packetService;
+
+        // Create a host service mock here. Must be replayed by tests once the
+        // expectations have been set up
+        hostService = createMock(HostService.class);
+        proxyArp.hostService = hostService;
+
+        createTopology();
+        proxyArp.deviceService = deviceService;
+        proxyArp.linkService = linkService;
+
+        proxyArp.activate();
+    }
+
+    /**
+     * Creates a fake topology to feed into the ARP module.
+     * <p/>
+     * The default topology is a unidirectional ring topology. Each switch has
+     * 3 ports. Ports 2 and 3 have the links to neighbor switches, and port 1
+     * is free (edge port).
+     */
+    private void createTopology() {
+        deviceService = createMock(DeviceService.class);
+        linkService = createMock(LinkService.class);
+
+        deviceService.addListener(anyObject(DeviceListener.class));
+        linkService.addListener(anyObject(LinkListener.class));
+
+        createDevices(NUM_DEVICES, NUM_PORTS_PER_DEVICE);
+        createLinks(NUM_DEVICES);
+    }
+
+    /**
+     * Creates the devices for the fake topology.
+     */
+    private void createDevices(int numDevices, int numPorts) {
+        List<Device> devices = new ArrayList<>();
+
+        for (int i = 1; i <= numDevices; i++) {
+            DeviceId devId = getDeviceId(i);
+            Device device = createMock(Device.class);
+            expect(device.id()).andReturn(devId).anyTimes();
+            replay(device);
+
+            devices.add(device);
+
+            List<Port> ports = new ArrayList<>();
+            for (int j = 1; j <= numPorts; j++) {
+                Port port = createMock(Port.class);
+                expect(port.number()).andReturn(PortNumber.portNumber(j)).anyTimes();
+                replay(port);
+                ports.add(port);
+            }
+
+            expect(deviceService.getPorts(devId)).andReturn(ports);
+        }
+
+        expect(deviceService.getDevices()).andReturn(devices);
+        replay(deviceService);
+    }
+
+    /**
+     * Creates the links for the fake topology.
+     * NB: Only unidirectional links are created, as for this purpose all we
+     * need is to occupy the ports with some link.
+     */
+    private void createLinks(int numDevices) {
+        List<Link> links = new ArrayList<Link>();
+
+        for (int i = 1; i <= numDevices; i++) {
+            ConnectPoint src = new ConnectPoint(
+                    getDeviceId(i),
+                    PortNumber.portNumber(2));
+            ConnectPoint dst = new ConnectPoint(
+                    getDeviceId((i + 1 > numDevices) ? 1 : i + 1),
+                    PortNumber.portNumber(3));
+
+            Link link = createMock(Link.class);
+            expect(link.src()).andReturn(src).anyTimes();
+            expect(link.dst()).andReturn(dst).anyTimes();
+            replay(link);
+
+            links.add(link);
+        }
+
+        expect(linkService.getLinks()).andReturn(links).anyTimes();
+        replay(linkService);
+    }
+
+    /**
+     * Tests {@link ProxyArpManager#known(IpPrefix)} in the case where the
+     * IP address is not known.
+     * Verifies the method returns false.
+     */
+    @Test
+    public void testNotKnown() {
+        expect(hostService.getHostsByIp(IP1)).andReturn(Collections.<Host>emptySet());
+        replay(hostService);
+
+        assertFalse(proxyArp.known(IP1));
+    }
+
+    /**
+     * Tests {@link ProxyArpManager#known(IpPrefix)} in the case where the
+     * IP address is known.
+     * Verifies the method returns true.
+     */
+    @Test
+    public void testKnown() {
+        Host host1 = createMock(Host.class);
+        Host host2 = createMock(Host.class);
+
+        expect(hostService.getHostsByIp(IP1))
+                .andReturn(Sets.newHashSet(host1, host2));
+        replay(hostService);
+
+        assertTrue(proxyArp.known(IP1));
+    }
+
+    /**
+     * Tests {@link ProxyArpManager#reply(Ethernet)} in the case where the
+     * destination host is known.
+     * Verifies the correct ARP reply is sent out the correct port.
+     */
+    @Test
+    public void testReplyKnown() {
+        Host replyer = new DefaultHost(PID, HID1, MAC1, VLAN1, LOC2,
+                Collections.singleton(IP1));
+
+        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, LOC1,
+                Collections.singleton(IP2));
+
+        expect(hostService.getHostsByIp(IpPrefix.valueOf(IP1.toOctets())))
+                .andReturn(Collections.singleton(replyer));
+        expect(hostService.getHost(HID2)).andReturn(requestor);
+
+        replay(hostService);
+
+        Ethernet arpRequest = buildArp(ARP.OP_REQUEST, MAC2, null, IP2, IP1);
+
+        proxyArp.reply(arpRequest);
+
+        assertEquals(1, packetService.packets.size());
+        Ethernet arpReply = buildArp(ARP.OP_REPLY, MAC1, MAC2, IP1, IP2);
+        verifyPacketOut(arpReply, LOC1, packetService.packets.get(0));
+    }
+
+    /**
+     * Tests {@link ProxyArpManager#reply(Ethernet)} in the case where the
+     * destination host is not known.
+     * Verifies the ARP request is flooded out the correct edge ports.
+     */
+    @Test
+    public void testReplyUnknown() {
+        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, LOC1,
+                Collections.singleton(IP2));
+
+        expect(hostService.getHostsByIp(IpPrefix.valueOf(IP1.toOctets())))
+                .andReturn(Collections.<Host>emptySet());
+        expect(hostService.getHost(HID2)).andReturn(requestor);
+
+        replay(hostService);
+
+        Ethernet arpRequest = buildArp(ARP.OP_REQUEST, MAC2, null, IP2, IP1);
+
+        proxyArp.reply(arpRequest);
+
+        verifyFlood(arpRequest);
+    }
+
+    /**
+     * Tests {@link ProxyArpManager#reply(Ethernet)} in the case where the
+     * destination host is known for that IP address, but is not on the same
+     * VLAN as the source host.
+     * Verifies the ARP request is flooded out the correct edge ports.
+     */
+    @Test
+    public void testReplyDifferentVlan() {
+        Host replyer = new DefaultHost(PID, HID1, MAC1, VLAN2, LOC2,
+                Collections.singleton(IP1));
+
+        Host requestor = new DefaultHost(PID, HID2, MAC2, VLAN1, LOC1,
+                Collections.singleton(IP2));
+
+        expect(hostService.getHostsByIp(IpPrefix.valueOf(IP1.toOctets())))
+                .andReturn(Collections.singleton(replyer));
+        expect(hostService.getHost(HID2)).andReturn(requestor);
+
+        replay(hostService);
+
+        Ethernet arpRequest = buildArp(ARP.OP_REQUEST, MAC2, null, IP2, IP1);
+
+        proxyArp.reply(arpRequest);
+
+        verifyFlood(arpRequest);
+    }
+
+    /**
+     * Tests {@link ProxyArpManager#forward(Ethernet)} in the case where the
+     * destination host is known.
+     * Verifies the correct ARP request is sent out the correct port.
+     */
+    @Test
+    public void testForwardToHost() {
+        Host host1 = new DefaultHost(PID, HID1, MAC1, VLAN1, LOC1,
+                Collections.singleton(IP1));
+
+        expect(hostService.getHost(HID1)).andReturn(host1);
+        replay(hostService);
+
+        Ethernet arpRequest = buildArp(ARP.OP_REPLY, MAC2, MAC1, IP2, IP1);
+
+        proxyArp.forward(arpRequest);
+
+        assertEquals(1, packetService.packets.size());
+        OutboundPacket packet = packetService.packets.get(0);
+
+        verifyPacketOut(arpRequest, LOC1, packet);
+    }
+
+    /**
+     * Tests {@link ProxyArpManager#forward(Ethernet)} in the case where the
+     * destination host is not known.
+     * Verifies the correct ARP request is flooded out the correct edge ports.
+     */
+    @Test
+    public void testForwardFlood() {
+        expect(hostService.getHost(HID1)).andReturn(null);
+        replay(hostService);
+
+        Ethernet arpRequest = buildArp(ARP.OP_REPLY, MAC2, MAC1, IP2, IP1);
+
+        proxyArp.forward(arpRequest);
+
+        verifyFlood(arpRequest);
+    }
+
+    /**
+     * Verifies that the given packet was flooded out all available edge ports.
+     *
+     * @param packet the packet that was expected to be flooded
+     */
+    private void verifyFlood(Ethernet packet) {
+        assertEquals(NUM_FLOOD_PORTS, packetService.packets.size());
+
+        Collections.sort(packetService.packets,
+            new Comparator<OutboundPacket>() {
+                @Override
+                public int compare(OutboundPacket o1, OutboundPacket o2) {
+                    return o1.sendThrough().uri().compareTo(o2.sendThrough().uri());
+                }
+            });
+
+        for (int i = 0; i < NUM_FLOOD_PORTS; i++) {
+            ConnectPoint cp = new ConnectPoint(getDeviceId(i + 1), PortNumber.portNumber(1));
+
+            OutboundPacket outboundPacket = packetService.packets.get(i);
+            verifyPacketOut(packet, cp, outboundPacket);
+        }
+    }
+
+    /**
+     * Verifies the given packet was sent out the given port.
+     *
+     * @param expected the packet that was expected to be sent
+     * @param outPort the port the packet was expected to be sent out
+     * @param actual the actual OutboundPacket to verify
+     */
+    private void verifyPacketOut(Ethernet expected, ConnectPoint outPort,
+            OutboundPacket actual) {
+        assertTrue(Arrays.equals(expected.serialize(), actual.data().array()));
+        assertEquals(1, actual.treatment().instructions().size());
+        assertEquals(outPort.deviceId(), actual.sendThrough());
+        Instruction instruction = actual.treatment().instructions().get(0);
+        assertTrue(instruction instanceof OutputInstruction);
+        assertEquals(outPort.port(), ((OutputInstruction) instruction).port());
+    }
+
+    /**
+     * Returns the device ID of the ith device.
+     *
+     * @param i device to get the ID of
+     * @return the device ID
+     */
+    private static DeviceId getDeviceId(int i) {
+        return DeviceId.deviceId("" + i);
+    }
+
+    /**
+     * Builds an ARP packet with the given parameters.
+     *
+     * @param opcode opcode of the ARP packet
+     * @param srcMac source MAC address
+     * @param dstMac destination MAC address, or null if this is a request
+     * @param srcIp source IP address
+     * @param dstIp destination IP address
+     * @return the ARP packet
+     */
+    private Ethernet buildArp(short opcode, MacAddress srcMac, MacAddress dstMac,
+            IpPrefix srcIp, IpPrefix dstIp) {
+        Ethernet eth = new Ethernet();
+
+        if (dstMac == null) {
+            eth.setDestinationMACAddress(MacAddress.BROADCAST_MAC);
+        } else {
+            eth.setDestinationMACAddress(dstMac.getAddress());
+        }
+
+        eth.setSourceMACAddress(srcMac.getAddress());
+        eth.setEtherType(Ethernet.TYPE_ARP);
+        eth.setVlanID(VLAN1.toShort());
+
+        ARP arp = new ARP();
+        arp.setOpCode(opcode);
+        arp.setProtocolType(ARP.PROTO_TYPE_IP);
+        arp.setHardwareType(ARP.HW_TYPE_ETHERNET);
+
+        arp.setProtocolAddressLength((byte) IpPrefix.INET_LEN);
+        arp.setHardwareAddressLength((byte) Ethernet.DATALAYER_ADDRESS_LENGTH);
+        arp.setSenderHardwareAddress(srcMac.getAddress());
+
+        if (dstMac == null) {
+            arp.setTargetHardwareAddress(MacAddress.ZERO_MAC_ADDRESS);
+        } else {
+            arp.setTargetHardwareAddress(dstMac.getAddress());
+        }
+
+        arp.setSenderProtocolAddress(srcIp.toOctets());
+        arp.setTargetProtocolAddress(dstIp.toOctets());
+
+        eth.setPayload(arp);
+        return eth;
+    }
+
+    /**
+     * Test PacketService implementation that simply stores OutboundPackets
+     * passed to {@link #emit(OutboundPacket)} for later verification.
+     */
+    class TestPacketService implements PacketService {
+
+        List<OutboundPacket> packets = new ArrayList<>();
+
+        @Override
+        public void addProcessor(PacketProcessor processor, int priority) {
+        }
+
+        @Override
+        public void removeProcessor(PacketProcessor processor) {
+        }
+
+        @Override
+        public void emit(OutboundPacket packet) {
+            packets.add(packet);
+        }
+    }
+}

@@ -65,47 +65,48 @@ public class PathManagerTest {
     @Test
     @Test
     public void infraToEdge() {
     public void infraToEdge() {
         DeviceId src = did("src");
         DeviceId src = did("src");
-        HostId dst = hid("dst");
+        HostId dst = hid("12:34:56:78:90:ab/1");
         fakeTopoMgr.paths.add(createPath("src", "middle", "edge"));
         fakeTopoMgr.paths.add(createPath("src", "middle", "edge"));
-        fakeHostMgr.hosts.put(dst, host("dst", "edge"));
+        fakeHostMgr.hosts.put(dst, host("12:34:56:78:90:ab/1", "edge"));
         Set<Path> paths = service.getPaths(src, dst);
         Set<Path> paths = service.getPaths(src, dst);
         validatePaths(paths, 1, 3, src, dst);
         validatePaths(paths, 1, 3, src, dst);
     }
     }
 
 
     @Test
     @Test
     public void edgeToInfra() {
     public void edgeToInfra() {
-        HostId src = hid("src");
+        HostId src = hid("12:34:56:78:90:ab/1");
         DeviceId dst = did("dst");
         DeviceId dst = did("dst");
         fakeTopoMgr.paths.add(createPath("edge", "middle", "dst"));
         fakeTopoMgr.paths.add(createPath("edge", "middle", "dst"));
-        fakeHostMgr.hosts.put(src, host("src", "edge"));
+        fakeHostMgr.hosts.put(src, host("12:34:56:78:90:ab/1", "edge"));
         Set<Path> paths = service.getPaths(src, dst);
         Set<Path> paths = service.getPaths(src, dst);
         validatePaths(paths, 1, 3, src, dst);
         validatePaths(paths, 1, 3, src, dst);
     }
     }
 
 
     @Test
     @Test
     public void edgeToEdge() {
     public void edgeToEdge() {
-        HostId src = hid("src");
-        HostId dst = hid("dst");
+        HostId src = hid("12:34:56:78:90:ab/1");
+        HostId dst = hid("12:34:56:78:90:ef/1");
         fakeTopoMgr.paths.add(createPath("srcEdge", "middle", "dstEdge"));
         fakeTopoMgr.paths.add(createPath("srcEdge", "middle", "dstEdge"));
-        fakeHostMgr.hosts.put(src, host("src", "srcEdge"));
-        fakeHostMgr.hosts.put(dst, host("dst", "dstEdge"));
+        fakeHostMgr.hosts.put(src, host("12:34:56:78:90:ab/1", "srcEdge"));
+        fakeHostMgr.hosts.put(dst, host("12:34:56:78:90:ef/1", "dstEdge"));
         Set<Path> paths = service.getPaths(src, dst);
         Set<Path> paths = service.getPaths(src, dst);
         validatePaths(paths, 1, 4, src, dst);
         validatePaths(paths, 1, 4, src, dst);
     }
     }
 
 
     @Test
     @Test
     public void edgeToEdgeDirect() {
     public void edgeToEdgeDirect() {
-        HostId src = hid("src");
-        HostId dst = hid("dst");
-        fakeHostMgr.hosts.put(src, host("src", "edge"));
-        fakeHostMgr.hosts.put(dst, host("dst", "edge"));
+        HostId src = hid("12:34:56:78:90:ab/1");
+        HostId dst = hid("12:34:56:78:90:ef/1");
+        fakeHostMgr.hosts.put(src, host("12:34:56:78:90:ab/1", "edge"));
+        fakeHostMgr.hosts.put(dst, host("12:34:56:78:90:ef/1", "edge"));
         Set<Path> paths = service.getPaths(src, dst);
         Set<Path> paths = service.getPaths(src, dst);
         validatePaths(paths, 1, 2, src, dst);
         validatePaths(paths, 1, 2, src, dst);
     }
     }
 
 
     @Test
     @Test
     public void noEdge() {
     public void noEdge() {
-        Set<Path> paths = service.getPaths(hid("src"), hid("dst"));
+        Set<Path> paths = service.getPaths(hid("12:34:56:78:90:ab/1"),
+                                           hid("12:34:56:78:90:ef/1"));
         assertTrue("there should be no paths", paths.isEmpty());
         assertTrue("there should be no paths", paths.isEmpty());
     }
     }
 
 

@@ -134,11 +134,11 @@ public class TopologyManagerTest {
                     service.isInfrastructure(topology, new ConnectPoint(did("a"), portNumber(3))));
                     service.isInfrastructure(topology, new ConnectPoint(did("a"), portNumber(3))));
 
 
         // One of these cannot be a broadcast point... or we have a loop...
         // One of these cannot be a broadcast point... or we have a loop...
-        assertFalse("should not be broadcast point",
-                    service.isBroadcastPoint(topology, new ConnectPoint(did("a"), portNumber(1))) &&
-                            service.isBroadcastPoint(topology, new ConnectPoint(did("b"), portNumber(1))) &&
-                            service.isBroadcastPoint(topology, new ConnectPoint(did("c"), portNumber(1))) &&
-                            service.isBroadcastPoint(topology, new ConnectPoint(did("d"), portNumber(1))));
+//        assertFalse("should not be broadcast point",
+//                    service.isBroadcastPoint(topology, new ConnectPoint(did("a"), portNumber(1))) &&
+//                            service.isBroadcastPoint(topology, new ConnectPoint(did("b"), portNumber(1))) &&
+//                            service.isBroadcastPoint(topology, new ConnectPoint(did("c"), portNumber(1))) &&
+//                            service.isBroadcastPoint(topology, new ConnectPoint(did("d"), portNumber(1))));
         assertTrue("should be broadcast point",
         assertTrue("should be broadcast point",
                    service.isBroadcastPoint(topology, new ConnectPoint(did("a"), portNumber(3))));
                    service.isBroadcastPoint(topology, new ConnectPoint(did("a"), portNumber(3))));
     }
     }

@@ -54,8 +54,18 @@
             <artifactId>org.apache.felix.scr.annotations</artifactId>
             <artifactId>org.apache.felix.scr.annotations</artifactId>
         </dependency>
         </dependency>
         <dependency>
         <dependency>
-          <groupId>de.javakaffee</groupId>
-          <artifactId>kryo-serializers</artifactId>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava-testlib</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+        </dependency>
+        <dependency>
+          <groupId>org.easymock</groupId>
+          <artifactId>easymock</artifactId>
+          <scope>test</scope>
         </dependency>
         </dependency>
     </dependencies>
     </dependencies>
 
 

-package org.onlab.onos.store.common.impl;
-
-import java.util.Map;
-
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.store.Timestamp;
-
-import com.google.common.collect.ImmutableMap;
-
-/**
- * Anti-Entropy advertisement message.
- * <p>
- * Message to advertise the information this node holds.
- *
- * @param <ID> ID type
- */
-public class AntiEntropyAdvertisement<ID> {
-
-    private final NodeId sender;
-    private final ImmutableMap<ID, Timestamp> advertisement;
-
-    /**
-     * Creates anti-entropy advertisement message.
-     *
-     * @param sender sender of this message
-     * @param advertisement timestamp information of the data sender holds
-     */
-    public AntiEntropyAdvertisement(NodeId sender, Map<ID, Timestamp> advertisement) {
-        this.sender = sender;
-        this.advertisement = ImmutableMap.copyOf(advertisement);
-    }
-
-    public NodeId sender() {
-        return sender;
-    }
-
-    public ImmutableMap<ID, Timestamp> advertisement() {
-        return advertisement;
-    }
-
-    // Default constructor for serializer
-    protected AntiEntropyAdvertisement() {
-        this.sender = null;
-        this.advertisement = null;
-    }
-}

-package org.onlab.onos.store.common.impl;
-
-import java.util.Map;
-import java.util.Set;
-
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.store.device.impl.VersionedValue;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
-/**
- * Anti-Entropy reply message.
- * <p>
- * Message to send in reply to advertisement or another reply.
- * Suggest to the sender about the more up-to-date data this node has,
- * and request for more recent data that the receiver has.
- */
-public class AntiEntropyReply<ID, V extends VersionedValue<?>> {
-
-    private final NodeId sender;
-    private final ImmutableMap<ID, V> suggestion;
-    private final ImmutableSet<ID> request;
-
-    /**
-     * Creates a reply to anti-entropy message.
-     *
-     * @param sender sender of this message
-     * @param suggestion collection of more recent values, sender had
-     * @param request Collection of identifiers
-     */
-    public AntiEntropyReply(NodeId sender,
-                            Map<ID, V> suggestion,
-                            Set<ID> request) {
-        this.sender = sender;
-        this.suggestion = ImmutableMap.copyOf(suggestion);
-        this.request = ImmutableSet.copyOf(request);
-    }
-
-    public NodeId sender() {
-        return sender;
-    }
-
-    /**
-     * Returns collection of values, which the recipient of this reply is likely
-     * to be missing or has outdated version.
-     *
-     * @return
-     */
-    public ImmutableMap<ID, V> suggestion() {
-        return suggestion;
-    }
-
-    /**
-     * Returns collection of identifier to request.
-     *
-     * @return collection of identifier to request
-     */
-    public ImmutableSet<ID> request() {
-        return request;
-    }
-
-    /**
-     * Checks if reply contains any suggestion or request.
-     *
-     * @return true if nothing is suggested and requested
-     */
-    public boolean isEmpty() {
-        return suggestion.isEmpty() && request.isEmpty();
-    }
-
-    // Default constructor for serializer
-    protected AntiEntropyReply() {
-        this.sender = null;
-        this.suggestion = null;
-        this.request = null;
-    }
-}

@@ -30,6 +30,7 @@ public final class Timestamped<T> {
 
 
     /**
     /**
      * Returns the value.
      * Returns the value.
+     *
      * @return value
      * @return value
      */
      */
     public T value() {
     public T value() {
@@ -38,6 +39,7 @@ public final class Timestamped<T> {
 
 
     /**
     /**
      * Returns the time stamp.
      * Returns the time stamp.
+     *
      * @return time stamp
      * @return time stamp
      */
      */
     public Timestamp timestamp() {
     public Timestamp timestamp() {
@@ -51,7 +53,16 @@ public final class Timestamped<T> {
      * @return true if this instance is newer.
      * @return true if this instance is newer.
      */
      */
     public boolean isNewer(Timestamped<T> other) {
     public boolean isNewer(Timestamped<T> other) {
-        return this.timestamp.compareTo(checkNotNull(other).timestamp()) > 0;
+        return isNewer(checkNotNull(other).timestamp());
+    }
+
+    /**
+     * Tests if this timestamp is newer thatn the specified timestamp.
+     * @param timestamp to compare agains
+     * @return true if this instance is newer
+     */
+    public boolean isNewer(Timestamp timestamp) {
+        return this.timestamp.compareTo(checkNotNull(timestamp)) > 0;
     }
     }
 
 
     @Override
     @Override

-package org.onlab.onos.store.device.impl;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.net.Device;
-import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.store.Timestamp;
-import org.onlab.onos.store.common.impl.AntiEntropyAdvertisement;
-
-// TODO DeviceID needs to be changed to something like (ProviderID, DeviceID)
-// TODO: Handle Port as part of these messages, or separate messages for Ports?
-
-public class DeviceAntiEntropyAdvertisement
-    extends AntiEntropyAdvertisement<DeviceId> {
-
-
-    public DeviceAntiEntropyAdvertisement(NodeId sender,
-            Map<DeviceId, Timestamp> advertisement) {
-        super(sender, advertisement);
-    }
-
-    // May need to add ProviderID, etc.
-    public static DeviceAntiEntropyAdvertisement create(
-            NodeId self,
-            Collection<VersionedValue<Device>> localValues) {
-
-        Map<DeviceId, Timestamp> ads = new HashMap<>(localValues.size());
-        for (VersionedValue<Device> e : localValues) {
-            ads.put(e.entity().id(), e.timestamp());
-        }
-        return new DeviceAntiEntropyAdvertisement(self, ads);
-    }
-
-    // For serializer
-    protected DeviceAntiEntropyAdvertisement() {}
-}

-package org.onlab.onos.store.device.impl;
-
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.net.Device;
-import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.store.Timestamp;
-import org.onlab.onos.store.common.impl.AntiEntropyReply;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
-public class DeviceAntiEntropyReply
-    extends AntiEntropyReply<DeviceId, VersionedValue<Device>> {
-
-
-    public DeviceAntiEntropyReply(NodeId sender,
-            Map<DeviceId, VersionedValue<Device>> suggestion,
-            Set<DeviceId> request) {
-        super(sender, suggestion, request);
-    }
-
-    /**
-     * Creates a reply to Anti-Entropy advertisement.
-     *
-     * @param advertisement to respond to
-     * @param self node identifier representing local node
-     * @param localValues local values held on this node
-     * @return reply message
-     */
-    public static DeviceAntiEntropyReply reply(
-            DeviceAntiEntropyAdvertisement advertisement,
-            NodeId self,
-            Collection<VersionedValue<Device>> localValues
-            ) {
-
-        ImmutableMap<DeviceId, Timestamp> ads = advertisement.advertisement();
-
-        ImmutableMap.Builder<DeviceId, VersionedValue<Device>>
-            sug = ImmutableMap.builder();
-
-        Set<DeviceId> req = new HashSet<>(ads.keySet());
-
-        for (VersionedValue<Device> e : localValues) {
-            final DeviceId id = e.entity().id();
-            final Timestamp local = e.timestamp();
-            final Timestamp theirs = ads.get(id);
-            if (theirs == null) {
-                // they don't have it, suggest
-                sug.put(id, e);
-                // don't need theirs
-                req.remove(id);
-            } else if (local.compareTo(theirs) < 0) {
-                // they got older one, suggest
-                sug.put(id, e);
-                // don't need theirs
-                req.remove(id);
-            } else if (local.equals(theirs)) {
-                // same, don't need theirs
-                req.remove(id);
-            }
-        }
-
-        return new DeviceAntiEntropyReply(self, sug.build(), req);
-    }
-
-    /**
-     * Creates a reply to request for values held locally.
-     *
-     * @param requests message containing the request
-     * @param self node identifier representing local node
-     * @param localValues local valeds held on this node
-     * @return reply message
-     */
-    public static DeviceAntiEntropyReply reply(
-            DeviceAntiEntropyReply requests,
-            NodeId self,
-            Map<DeviceId, VersionedValue<Device>> localValues
-            ) {
-
-        Set<DeviceId> reqs = requests.request();
-
-        Map<DeviceId, VersionedValue<Device>> requested = new HashMap<>(reqs.size());
-        for (DeviceId id : reqs) {
-            final VersionedValue<Device> value = localValues.get(id);
-            if (value != null) {
-                requested.put(id, value);
-            }
-        }
-
-        Set<DeviceId> empty = ImmutableSet.of();
-        return new DeviceAntiEntropyReply(self, requested, empty);
-    }
-
-    // For serializer
-    protected DeviceAntiEntropyReply() {}
-}

+package org.onlab.onos.store.device.impl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static org.onlab.onos.net.DefaultAnnotations.union;
+
+import java.util.Collections;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.SparseAnnotations;
+import org.onlab.onos.net.device.DefaultDeviceDescription;
+import org.onlab.onos.net.device.DefaultPortDescription;
+import org.onlab.onos.net.device.DeviceDescription;
+import org.onlab.onos.net.device.PortDescription;
+import org.onlab.onos.store.Timestamp;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+/*
+ * Collection of Description of a Device and Ports, given from a Provider.
+ */
+class DeviceDescriptions {
+
+    private volatile Timestamped<DeviceDescription> deviceDesc;
+
+    private final ConcurrentMap<PortNumber, Timestamped<PortDescription>> portDescs;
+
+    public DeviceDescriptions(Timestamped<DeviceDescription> desc) {
+        this.deviceDesc = checkNotNull(desc);
+        this.portDescs = new ConcurrentHashMap<>();
+    }
+
+    public Timestamp getLatestTimestamp() {
+        Timestamp latest = deviceDesc.timestamp();
+        for (Timestamped<PortDescription> desc : portDescs.values()) {
+            if (desc.timestamp().compareTo(latest) > 0) {
+                latest = desc.timestamp();
+            }
+        }
+        return latest;
+    }
+
+    public Timestamped<DeviceDescription> getDeviceDesc() {
+        return deviceDesc;
+    }
+
+    public Timestamped<PortDescription> getPortDesc(PortNumber number) {
+        return portDescs.get(number);
+    }
+
+    public Map<PortNumber, Timestamped<PortDescription>> getPortDescs() {
+        return Collections.unmodifiableMap(portDescs);
+    }
+
+    /**
+     * Puts DeviceDescription, merging annotations as necessary.
+     *
+     * @param newDesc new DeviceDescription
+     */
+    public void putDeviceDesc(Timestamped<DeviceDescription> newDesc) {
+        Timestamped<DeviceDescription> oldOne = deviceDesc;
+        Timestamped<DeviceDescription> newOne = newDesc;
+        if (oldOne != null) {
+            SparseAnnotations merged = union(oldOne.value().annotations(),
+                                             newDesc.value().annotations());
+            newOne = new Timestamped<DeviceDescription>(
+                    new DefaultDeviceDescription(newDesc.value(), merged),
+                    newDesc.timestamp());
+        }
+        deviceDesc = newOne;
+    }
+
+    /**
+     * Puts PortDescription, merging annotations as necessary.
+     *
+     * @param newDesc new PortDescription
+     */
+    public void putPortDesc(Timestamped<PortDescription> newDesc) {
+        Timestamped<PortDescription> oldOne = portDescs.get(newDesc.value().portNumber());
+        Timestamped<PortDescription> newOne = newDesc;
+        if (oldOne != null) {
+            SparseAnnotations merged = union(oldOne.value().annotations(),
+                                             newDesc.value().annotations());
+            newOne = new Timestamped<PortDescription>(
+                    new DefaultPortDescription(newDesc.value(), merged),
+                    newDesc.timestamp());
+        }
+        portDescs.put(newOne.value().portNumber(), newOne);
+    }
+}

 package org.onlab.onos.store.device.impl;
 package org.onlab.onos.store.device.impl;
 
 
+import com.google.common.base.Function;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Sets;
 import com.google.common.collect.Sets;
 
 
-import org.apache.commons.lang3.concurrent.ConcurrentException;
-import org.apache.commons.lang3.concurrent.ConcurrentInitializer;
+import org.apache.commons.lang3.RandomUtils;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -14,6 +14,8 @@ import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.onos.cluster.ClusterService;
 import org.onlab.onos.cluster.ClusterService;
+import org.onlab.onos.cluster.ControllerNode;
+import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.net.AnnotationsUtil;
 import org.onlab.onos.net.AnnotationsUtil;
 import org.onlab.onos.net.DefaultAnnotations;
 import org.onlab.onos.net.DefaultAnnotations;
 import org.onlab.onos.net.DefaultDevice;
 import org.onlab.onos.net.DefaultDevice;
@@ -23,9 +25,6 @@ import org.onlab.onos.net.Device.Type;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Port;
 import org.onlab.onos.net.Port;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.PortNumber;
-import org.onlab.onos.net.SparseAnnotations;
-import org.onlab.onos.net.device.DefaultDeviceDescription;
-import org.onlab.onos.net.device.DefaultPortDescription;
 import org.onlab.onos.net.device.DeviceDescription;
 import org.onlab.onos.net.device.DeviceDescription;
 import org.onlab.onos.net.device.DeviceEvent;
 import org.onlab.onos.net.device.DeviceEvent;
 import org.onlab.onos.net.device.DeviceStore;
 import org.onlab.onos.net.device.DeviceStore;
@@ -38,18 +37,22 @@ import org.onlab.onos.store.Timestamp;
 import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
 import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
 import org.onlab.onos.store.cluster.messaging.ClusterMessage;
 import org.onlab.onos.store.cluster.messaging.ClusterMessage;
 import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
 import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
-import org.onlab.onos.store.common.impl.MastershipBasedTimestamp;
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
 import org.onlab.onos.store.common.impl.Timestamped;
 import org.onlab.onos.store.common.impl.Timestamped;
-import org.onlab.onos.store.serializers.KryoPoolUtil;
+import org.onlab.onos.store.device.impl.peermsg.DeviceAntiEntropyAdvertisement;
+import org.onlab.onos.store.device.impl.peermsg.DeviceFragmentId;
+import org.onlab.onos.store.device.impl.peermsg.PortFragmentId;
 import org.onlab.onos.store.serializers.KryoSerializer;
 import org.onlab.onos.store.serializers.KryoSerializer;
-import org.onlab.onos.store.serializers.MastershipBasedTimestampSerializer;
+import org.onlab.onos.store.serializers.DistributedStoreSerializers;
 import org.onlab.util.KryoPool;
 import org.onlab.util.KryoPool;
 import org.onlab.util.NewConcurrentHashMap;
 import org.onlab.util.NewConcurrentHashMap;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
 import java.io.IOException;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.Iterator;
 import java.util.List;
 import java.util.List;
@@ -57,19 +60,21 @@ import java.util.Map;
 import java.util.Map.Entry;
 import java.util.Map.Entry;
 import java.util.Objects;
 import java.util.Objects;
 import java.util.Set;
 import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
 
 
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Predicates.notNull;
 import static com.google.common.base.Predicates.notNull;
+import static org.onlab.onos.cluster.ControllerNodeToNodeId.toNodeId;
 import static org.onlab.onos.net.device.DeviceEvent.Type.*;
 import static org.onlab.onos.net.device.DeviceEvent.Type.*;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.apache.commons.lang3.concurrent.ConcurrentUtils.createIfAbsentUnchecked;
 import static org.apache.commons.lang3.concurrent.ConcurrentUtils.createIfAbsentUnchecked;
 import static org.onlab.onos.net.DefaultAnnotations.merge;
 import static org.onlab.onos.net.DefaultAnnotations.merge;
-import static org.onlab.onos.net.DefaultAnnotations.union;
 import static com.google.common.base.Verify.verify;
 import static com.google.common.base.Verify.verify;
+import static org.onlab.util.Tools.namedThreads;
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.onos.store.device.impl.GossipDeviceStoreMessageSubjects.DEVICE_ADVERTISE;
 
 
 // TODO: give me a better name
 // TODO: give me a better name
 /**
 /**
@@ -86,8 +91,9 @@ public class GossipDeviceStore
 
 
     public static final String DEVICE_NOT_FOUND = "Device with ID %s not found";
     public static final String DEVICE_NOT_FOUND = "Device with ID %s not found";
 
 
-    // TODO: Check if inner Map can be replaced with plain Map
+    // TODO: Check if inner Map can be replaced with plain Map.
     // innerMap is used to lock a Device, thus instance should never be replaced.
     // innerMap is used to lock a Device, thus instance should never be replaced.
+
     // collection of Description given from various providers
     // collection of Description given from various providers
     private final ConcurrentMap<DeviceId,
     private final ConcurrentMap<DeviceId,
                             ConcurrentMap<ProviderId, DeviceDescriptions>>
                             ConcurrentMap<ProviderId, DeviceDescriptions>>
@@ -113,25 +119,27 @@ public class GossipDeviceStore
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected ClusterService clusterService;
     protected ClusterService clusterService;
 
 
-    private static final KryoSerializer SERIALIZER = new KryoSerializer() {
+    protected static final KryoSerializer SERIALIZER = new KryoSerializer() {
         @Override
         @Override
         protected void setupKryoPool() {
         protected void setupKryoPool() {
             serializerPool = KryoPool.newBuilder()
             serializerPool = KryoPool.newBuilder()
-                    .register(KryoPoolUtil.API)
+                    .register(DistributedStoreSerializers.COMMON)
+
                     .register(InternalDeviceEvent.class, new InternalDeviceEventSerializer())
                     .register(InternalDeviceEvent.class, new InternalDeviceEventSerializer())
                     .register(InternalDeviceOfflineEvent.class, new InternalDeviceOfflineEventSerializer())
                     .register(InternalDeviceOfflineEvent.class, new InternalDeviceOfflineEventSerializer())
                     .register(InternalDeviceRemovedEvent.class)
                     .register(InternalDeviceRemovedEvent.class)
                     .register(InternalPortEvent.class, new InternalPortEventSerializer())
                     .register(InternalPortEvent.class, new InternalPortEventSerializer())
                     .register(InternalPortStatusEvent.class, new InternalPortStatusEventSerializer())
                     .register(InternalPortStatusEvent.class, new InternalPortStatusEventSerializer())
-                    .register(Timestamp.class)
-                    .register(Timestamped.class)
-                    .register(MastershipBasedTimestamp.class, new MastershipBasedTimestampSerializer())
+                    .register(DeviceAntiEntropyAdvertisement.class)
+                    .register(DeviceFragmentId.class)
+                    .register(PortFragmentId.class)
                     .build()
                     .build()
                     .populate(1);
                     .populate(1);
         }
         }
-
     };
     };
 
 
+    private ScheduledExecutorService executor;
+
     @Activate
     @Activate
     public void activate() {
     public void activate() {
         clusterCommunicator.addSubscriber(
         clusterCommunicator.addSubscriber(
@@ -144,11 +152,35 @@ public class GossipDeviceStore
                 GossipDeviceStoreMessageSubjects.PORT_UPDATE, new InternalPortEventListener());
                 GossipDeviceStoreMessageSubjects.PORT_UPDATE, new InternalPortEventListener());
         clusterCommunicator.addSubscriber(
         clusterCommunicator.addSubscriber(
                 GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE, new InternalPortStatusEventListener());
                 GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE, new InternalPortStatusEventListener());
+        clusterCommunicator.addSubscriber(
+                GossipDeviceStoreMessageSubjects.DEVICE_ADVERTISE, new InternalDeviceAdvertisementListener());
+
+        executor =
+                newSingleThreadScheduledExecutor(namedThreads("anti-entropy-%d"));
+
+        // TODO: Make these configurable
+        long initialDelaySec = 5;
+        long periodSec = 5;
+        // start anti-entropy thread
+        executor.scheduleAtFixedRate(new SendAdvertisementTask(),
+                    initialDelaySec, periodSec, TimeUnit.SECONDS);
+
         log.info("Started");
         log.info("Started");
     }
     }
 
 
     @Deactivate
     @Deactivate
     public void deactivate() {
     public void deactivate() {
+
+        executor.shutdownNow();
+        try {
+            boolean timedout = executor.awaitTermination(5, TimeUnit.SECONDS);
+            if (timedout) {
+                log.error("Timeout during executor shutdown");
+            }
+        } catch (InterruptedException e) {
+            log.error("Error during executor shutdown", e);
+        }
+
         deviceDescs.clear();
         deviceDescs.clear();
         devices.clear();
         devices.clear();
         devicePorts.clear();
         devicePorts.clear();
@@ -175,14 +207,19 @@ public class GossipDeviceStore
     public synchronized DeviceEvent createOrUpdateDevice(ProviderId providerId,
     public synchronized DeviceEvent createOrUpdateDevice(ProviderId providerId,
                                      DeviceId deviceId,
                                      DeviceId deviceId,
                                      DeviceDescription deviceDescription) {
                                      DeviceDescription deviceDescription) {
-        Timestamp newTimestamp = clockService.getTimestamp(deviceId);
+        final Timestamp newTimestamp = clockService.getTimestamp(deviceId);
         final Timestamped<DeviceDescription> deltaDesc = new Timestamped<>(deviceDescription, newTimestamp);
         final Timestamped<DeviceDescription> deltaDesc = new Timestamped<>(deviceDescription, newTimestamp);
-        DeviceEvent event = createOrUpdateDeviceInternal(providerId, deviceId, deltaDesc);
+        final DeviceEvent event;
+        final Timestamped<DeviceDescription> mergedDesc;
+        synchronized (getDeviceDescriptions(deviceId)) {
+            event = createOrUpdateDeviceInternal(providerId, deviceId, deltaDesc);
+            mergedDesc = getDeviceDescriptions(deviceId).get(providerId).getDeviceDesc();
+        }
         if (event != null) {
         if (event != null) {
             log.info("Notifying peers of a device update topology event for providerId: {} and deviceId: {}",
             log.info("Notifying peers of a device update topology event for providerId: {} and deviceId: {}",
                 providerId, deviceId);
                 providerId, deviceId);
             try {
             try {
-                notifyPeers(new InternalDeviceEvent(providerId, deviceId, deltaDesc));
+                notifyPeers(new InternalDeviceEvent(providerId, deviceId, mergedDesc));
             } catch (IOException e) {
             } catch (IOException e) {
                 log.error("Failed to notify peers of a device update topology event for providerId: "
                 log.error("Failed to notify peers of a device update topology event for providerId: "
                         + providerId + " and deviceId: " + deviceId, e);
                         + providerId + " and deviceId: " + deviceId, e);
@@ -286,8 +323,8 @@ public class GossipDeviceStore
 
 
     @Override
     @Override
     public DeviceEvent markOffline(DeviceId deviceId) {
     public DeviceEvent markOffline(DeviceId deviceId) {
-        Timestamp timestamp = clockService.getTimestamp(deviceId);
-        DeviceEvent event = markOfflineInternal(deviceId, timestamp);
+        final Timestamp timestamp = clockService.getTimestamp(deviceId);
+        final DeviceEvent event = markOfflineInternal(deviceId, timestamp);
         if (event != null) {
         if (event != null) {
             log.info("Notifying peers of a device offline topology event for deviceId: {}",
             log.info("Notifying peers of a device offline topology event for deviceId: {}",
                     deviceId);
                     deviceId);
@@ -359,17 +396,33 @@ public class GossipDeviceStore
     public synchronized List<DeviceEvent> updatePorts(ProviderId providerId,
     public synchronized List<DeviceEvent> updatePorts(ProviderId providerId,
                                        DeviceId deviceId,
                                        DeviceId deviceId,
                                        List<PortDescription> portDescriptions) {
                                        List<PortDescription> portDescriptions) {
-        Timestamp newTimestamp = clockService.getTimestamp(deviceId);
-
-        Timestamped<List<PortDescription>> timestampedPortDescriptions =
-            new Timestamped<>(portDescriptions, newTimestamp);
 
 
-        List<DeviceEvent> events = updatePortsInternal(providerId, deviceId, timestampedPortDescriptions);
+        final Timestamp newTimestamp = clockService.getTimestamp(deviceId);
+
+        final Timestamped<List<PortDescription>> timestampedInput
+                = new Timestamped<>(portDescriptions, newTimestamp);
+        final List<DeviceEvent> events;
+        final Timestamped<List<PortDescription>> merged;
+
+        synchronized (getDeviceDescriptions(deviceId)) {
+            events = updatePortsInternal(providerId, deviceId, timestampedInput);
+            final DeviceDescriptions descs = getDeviceDescriptions(deviceId).get(providerId);
+            List<PortDescription> mergedList =
+                    FluentIterable.from(portDescriptions)
+                .transform(new Function<PortDescription, PortDescription>() {
+                    @Override
+                    public PortDescription apply(PortDescription input) {
+                        // lookup merged port description
+                        return descs.getPortDesc(input.portNumber()).value();
+                    }
+                }).toList();
+            merged = new Timestamped<List<PortDescription>>(mergedList, newTimestamp);
+        }
         if (!events.isEmpty()) {
         if (!events.isEmpty()) {
             log.info("Notifying peers of a port update topology event for providerId: {} and deviceId: {}",
             log.info("Notifying peers of a port update topology event for providerId: {} and deviceId: {}",
                     providerId, deviceId);
                     providerId, deviceId);
             try {
             try {
-                notifyPeers(new InternalPortEvent(providerId, deviceId, timestampedPortDescriptions));
+                notifyPeers(new InternalPortEvent(providerId, deviceId, merged));
             } catch (IOException e) {
             } catch (IOException e) {
                 log.error("Failed to notify peers of a port update topology event or providerId: "
                 log.error("Failed to notify peers of a port update topology event or providerId: "
                     + providerId + " and deviceId: " + deviceId, e);
                     + providerId + " and deviceId: " + deviceId, e);
@@ -496,16 +549,25 @@ public class GossipDeviceStore
     }
     }
 
 
     @Override
     @Override
-    public synchronized DeviceEvent updatePortStatus(ProviderId providerId, DeviceId deviceId,
-            PortDescription portDescription) {
-        Timestamp newTimestamp = clockService.getTimestamp(deviceId);
-        final Timestamped<PortDescription> deltaDesc = new Timestamped<>(portDescription, newTimestamp);
-        DeviceEvent event = updatePortStatusInternal(providerId, deviceId, deltaDesc);
+    public synchronized DeviceEvent updatePortStatus(ProviderId providerId,
+                                                     DeviceId deviceId,
+                                                     PortDescription portDescription) {
+
+        final Timestamp newTimestamp = clockService.getTimestamp(deviceId);
+        final Timestamped<PortDescription> deltaDesc
+            = new Timestamped<>(portDescription, newTimestamp);
+        final DeviceEvent event;
+        final Timestamped<PortDescription> mergedDesc;
+        synchronized (getDeviceDescriptions(deviceId)) {
+            event = updatePortStatusInternal(providerId, deviceId, deltaDesc);
+            mergedDesc = getDeviceDescriptions(deviceId).get(providerId)
+                            .getPortDesc(portDescription.portNumber());
+        }
         if (event != null) {
         if (event != null) {
             log.info("Notifying peers of a port status update topology event for providerId: {} and deviceId: {}",
             log.info("Notifying peers of a port status update topology event for providerId: {} and deviceId: {}",
                         providerId, deviceId);
                         providerId, deviceId);
             try {
             try {
-                notifyPeers(new InternalPortStatusEvent(providerId, deviceId, deltaDesc));
+                notifyPeers(new InternalPortStatusEvent(providerId, deviceId, mergedDesc));
             } catch (IOException e) {
             } catch (IOException e) {
                 log.error("Failed to notify peers of a port status update topology event or providerId: "
                 log.error("Failed to notify peers of a port status update topology event or providerId: "
                         + providerId + " and deviceId: " + deviceId, e);
                         + providerId + " and deviceId: " + deviceId, e);
@@ -543,14 +605,14 @@ public class GossipDeviceStore
 
 
             final Timestamped<PortDescription> existingPortDesc = descs.getPortDesc(number);
             final Timestamped<PortDescription> existingPortDesc = descs.getPortDesc(number);
             if (existingPortDesc == null ||
             if (existingPortDesc == null ||
-                deltaDesc == existingPortDesc ||
                 deltaDesc.isNewer(existingPortDesc)) {
                 deltaDesc.isNewer(existingPortDesc)) {
                 // on new port or valid update
                 // on new port or valid update
                 // update description
                 // update description
                 descs.putPortDesc(deltaDesc);
                 descs.putPortDesc(deltaDesc);
                 newPort = composePort(device, number, descsMap);
                 newPort = composePort(device, number, descsMap);
             } else {
             } else {
-                // outdated event, ignored.
+                // same or outdated event, ignored.
+                log.trace("ignore same or outdated {} >= {}", existingPortDesc, deltaDesc);
                 return null;
                 return null;
             }
             }
 
 
@@ -627,6 +689,14 @@ public class GossipDeviceStore
         }
         }
     }
     }
 
 
+    /**
+     * Checks if given timestamp is superseded by removal request
+     * with more recent timestamp.
+     *
+     * @param deviceId identifier of a device
+     * @param timestampToCheck timestamp of an event to check
+     * @return true if device is already removed
+     */
     private boolean isDeviceRemoved(DeviceId deviceId, Timestamp timestampToCheck) {
     private boolean isDeviceRemoved(DeviceId deviceId, Timestamp timestampToCheck) {
         Timestamp removalTimestamp = removalRequest.get(deviceId);
         Timestamp removalTimestamp = removalRequest.get(deviceId);
         if (removalTimestamp != null &&
         if (removalTimestamp != null &&
@@ -645,7 +715,7 @@ public class GossipDeviceStore
      * @return Device instance
      * @return Device instance
      */
      */
     private Device composeDevice(DeviceId deviceId,
     private Device composeDevice(DeviceId deviceId,
-            ConcurrentMap<ProviderId, DeviceDescriptions> providerDescs) {
+            Map<ProviderId, DeviceDescriptions> providerDescs) {
 
 
         checkArgument(!providerDescs.isEmpty(), "No Device descriptions supplied");
         checkArgument(!providerDescs.isEmpty(), "No Device descriptions supplied");
 
 
@@ -667,7 +737,7 @@ public class GossipDeviceStore
                 continue;
                 continue;
             }
             }
             // TODO: should keep track of Description timestamp
             // TODO: should keep track of Description timestamp
-            // and only merge conflicting keys when timestamp is newer
+            // and only merge conflicting keys when timestamp is newer.
             // Currently assuming there will never be a key conflict between
             // Currently assuming there will never be a key conflict between
             // providers
             // providers
 
 
@@ -708,7 +778,7 @@ public class GossipDeviceStore
                 continue;
                 continue;
             }
             }
             // TODO: should keep track of Description timestamp
             // TODO: should keep track of Description timestamp
-            // and only merge conflicting keys when timestamp is newer
+            // and only merge conflicting keys when timestamp is newer.
             // Currently assuming there will never be a key conflict between
             // Currently assuming there will never be a key conflict between
             // providers
             // providers
 
 
@@ -745,129 +815,312 @@ public class GossipDeviceStore
         return providerDescs.get(pid);
         return providerDescs.get(pid);
     }
     }
 
 
-    public static final class InitDeviceDescs
-        implements ConcurrentInitializer<DeviceDescriptions> {
+    // TODO: should we be throwing exception?
+    private void unicastMessage(NodeId recipient, MessageSubject subject, Object event) throws IOException {
+        ClusterMessage message = new ClusterMessage(
+                clusterService.getLocalNode().id(),
+                subject,
+                SERIALIZER.encode(event));
+        clusterCommunicator.unicast(message, recipient);
+    }
 
 
-        private final Timestamped<DeviceDescription> deviceDesc;
+    // TODO: should we be throwing exception?
+    private void broadcastMessage(MessageSubject subject, Object event) throws IOException {
+        ClusterMessage message = new ClusterMessage(
+                clusterService.getLocalNode().id(),
+                subject,
+                SERIALIZER.encode(event));
+        clusterCommunicator.broadcast(message);
+    }
 
 
-        public InitDeviceDescs(Timestamped<DeviceDescription> deviceDesc) {
-            this.deviceDesc = checkNotNull(deviceDesc);
-        }
-        @Override
-        public DeviceDescriptions get() throws ConcurrentException {
-            return new DeviceDescriptions(deviceDesc);
-        }
+    private void notifyPeers(InternalDeviceEvent event) throws IOException {
+        broadcastMessage(GossipDeviceStoreMessageSubjects.DEVICE_UPDATE, event);
     }
     }
 
 
+    private void notifyPeers(InternalDeviceOfflineEvent event) throws IOException {
+        broadcastMessage(GossipDeviceStoreMessageSubjects.DEVICE_OFFLINE, event);
+    }
 
 
-    /**
-     * Collection of Description of a Device and it's Ports given from a Provider.
-     */
-    public static class DeviceDescriptions {
+    private void notifyPeers(InternalDeviceRemovedEvent event) throws IOException {
+        broadcastMessage(GossipDeviceStoreMessageSubjects.DEVICE_REMOVED, event);
+    }
 
 
-        private final AtomicReference<Timestamped<DeviceDescription>> deviceDesc;
-        private final ConcurrentMap<PortNumber, Timestamped<PortDescription>> portDescs;
+    private void notifyPeers(InternalPortEvent event) throws IOException {
+        broadcastMessage(GossipDeviceStoreMessageSubjects.PORT_UPDATE, event);
+    }
 
 
-        public DeviceDescriptions(Timestamped<DeviceDescription> desc) {
-            this.deviceDesc = new AtomicReference<>(checkNotNull(desc));
-            this.portDescs = new ConcurrentHashMap<>();
-        }
+    private void notifyPeers(InternalPortStatusEvent event) throws IOException {
+        broadcastMessage(GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE, event);
+    }
 
 
-        Timestamp getLatestTimestamp() {
-            Timestamp latest = deviceDesc.get().timestamp();
-            for (Timestamped<PortDescription> desc : portDescs.values()) {
-                if (desc.timestamp().compareTo(latest) > 0) {
-                    latest = desc.timestamp();
-                }
-            }
-            return latest;
+    private void notifyPeer(NodeId recipient, InternalDeviceEvent event) {
+        try {
+            unicastMessage(recipient, GossipDeviceStoreMessageSubjects.DEVICE_UPDATE, event);
+        } catch (IOException e) {
+            log.error("Failed to send" + event + " to " + recipient, e);
         }
         }
+    }
 
 
-        public Timestamped<DeviceDescription> getDeviceDesc() {
-            return deviceDesc.get();
+    private void notifyPeer(NodeId recipient, InternalDeviceOfflineEvent event) {
+        try {
+            unicastMessage(recipient, GossipDeviceStoreMessageSubjects.DEVICE_OFFLINE, event);
+        } catch (IOException e) {
+            log.error("Failed to send" + event + " to " + recipient, e);
         }
         }
+    }
 
 
-        public Timestamped<PortDescription> getPortDesc(PortNumber number) {
-            return portDescs.get(number);
+    private void notifyPeer(NodeId recipient, InternalDeviceRemovedEvent event) {
+        try {
+            unicastMessage(recipient, GossipDeviceStoreMessageSubjects.DEVICE_REMOVED, event);
+        } catch (IOException e) {
+            log.error("Failed to send" + event + " to " + recipient, e);
         }
         }
+    }
 
 
-        /**
-         * Puts DeviceDescription, merging annotations as necessary.
-         *
-         * @param newDesc new DeviceDescription
-         * @return previous DeviceDescription
-         */
-        public synchronized Timestamped<DeviceDescription> putDeviceDesc(Timestamped<DeviceDescription> newDesc) {
-            Timestamped<DeviceDescription> oldOne = deviceDesc.get();
-            Timestamped<DeviceDescription> newOne = newDesc;
-            if (oldOne != null) {
-                SparseAnnotations merged = union(oldOne.value().annotations(),
-                                                 newDesc.value().annotations());
-                newOne = new Timestamped<DeviceDescription>(
-                        new DefaultDeviceDescription(newDesc.value(), merged),
-                        newDesc.timestamp());
-            }
-            return deviceDesc.getAndSet(newOne);
-        }
-
-        /**
-         * Puts PortDescription, merging annotations as necessary.
-         *
-         * @param newDesc new PortDescription
-         * @return previous PortDescription
-         */
-        public synchronized Timestamped<PortDescription> putPortDesc(Timestamped<PortDescription> newDesc) {
-            Timestamped<PortDescription> oldOne = portDescs.get(newDesc.value().portNumber());
-            Timestamped<PortDescription> newOne = newDesc;
-            if (oldOne != null) {
-                SparseAnnotations merged = union(oldOne.value().annotations(),
-                                                 newDesc.value().annotations());
-                newOne = new Timestamped<PortDescription>(
-                        new DefaultPortDescription(newDesc.value(), merged),
-                        newDesc.timestamp());
-            }
-            return portDescs.put(newOne.value().portNumber(), newOne);
+    private void notifyPeer(NodeId recipient, InternalPortEvent event) {
+        try {
+            unicastMessage(recipient, GossipDeviceStoreMessageSubjects.PORT_UPDATE, event);
+        } catch (IOException e) {
+            log.error("Failed to send" + event + " to " + recipient, e);
         }
         }
     }
     }
 
 
-    private void notifyPeers(InternalDeviceEvent event) throws IOException {
-        ClusterMessage message = new ClusterMessage(
-                clusterService.getLocalNode().id(),
-                GossipDeviceStoreMessageSubjects.DEVICE_UPDATE,
-                SERIALIZER.encode(event));
-        clusterCommunicator.broadcast(message);
+    private void notifyPeer(NodeId recipient, InternalPortStatusEvent event) {
+        try {
+            unicastMessage(recipient, GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE, event);
+        } catch (IOException e) {
+            log.error("Failed to send" + event + " to " + recipient, e);
+        }
     }
     }
 
 
-    private void notifyPeers(InternalDeviceOfflineEvent event) throws IOException {
-        ClusterMessage message = new ClusterMessage(
-                clusterService.getLocalNode().id(),
-                GossipDeviceStoreMessageSubjects.DEVICE_OFFLINE,
-                SERIALIZER.encode(event));
-        clusterCommunicator.broadcast(message);
+    private DeviceAntiEntropyAdvertisement createAdvertisement() {
+        final NodeId self = clusterService.getLocalNode().id();
+
+        Map<DeviceFragmentId, Timestamp> devices = new HashMap<>(deviceDescs.size());
+        final int portsPerDevice = 8; // random guess to minimize reallocation
+        Map<PortFragmentId, Timestamp> ports = new HashMap<>(devices.size() * portsPerDevice);
+        Map<DeviceId, Timestamp> offline = new HashMap<>(devices.size());
+
+        for (Entry<DeviceId, ConcurrentMap<ProviderId, DeviceDescriptions>>
+            provs : deviceDescs.entrySet()) {
+
+            final DeviceId deviceId = provs.getKey();
+            final ConcurrentMap<ProviderId, DeviceDescriptions> devDescs = provs.getValue();
+            synchronized (devDescs) {
+
+                offline.put(deviceId, this.offline.get(deviceId));
+
+                for (Entry<ProviderId, DeviceDescriptions>
+                        prov : devDescs.entrySet()) {
+
+                    final ProviderId provId = prov.getKey();
+                    final DeviceDescriptions descs = prov.getValue();
+
+                    devices.put(new DeviceFragmentId(deviceId, provId),
+                            descs.getDeviceDesc().timestamp());
+
+                    for (Entry<PortNumber, Timestamped<PortDescription>>
+                            portDesc : descs.getPortDescs().entrySet()) {
+
+                        final PortNumber number = portDesc.getKey();
+                        ports.put(new PortFragmentId(deviceId, provId, number),
+                                portDesc.getValue().timestamp());
+                    }
+                }
+            }
+        }
+
+        return new DeviceAntiEntropyAdvertisement(self, devices, ports, offline);
     }
     }
 
 
-    private void notifyPeers(InternalDeviceRemovedEvent event) throws IOException {
-        ClusterMessage message = new ClusterMessage(
-                clusterService.getLocalNode().id(),
-                GossipDeviceStoreMessageSubjects.DEVICE_REMOVED,
-                SERIALIZER.encode(event));
-        clusterCommunicator.broadcast(message);
+    /**
+     * Responds to anti-entropy advertisement message.
+     * <P>
+     * Notify sender about out-dated information using regular replication message.
+     * Send back advertisement to sender if not in sync.
+     *
+     * @param advertisement to respond to
+     */
+    private void handleAdvertisement(DeviceAntiEntropyAdvertisement advertisement) {
+
+        final NodeId sender = advertisement.sender();
+
+        Map<DeviceFragmentId, Timestamp> devAds = new HashMap<>(advertisement.deviceFingerPrints());
+        Map<PortFragmentId, Timestamp> portAds = new HashMap<>(advertisement.ports());
+        Map<DeviceId, Timestamp> offlineAds = new HashMap<>(advertisement.offline());
+
+        // Fragments to request
+        Collection<DeviceFragmentId> reqDevices = new ArrayList<>();
+        Collection<PortFragmentId> reqPorts = new ArrayList<>();
+
+        for (Entry<DeviceId, ConcurrentMap<ProviderId, DeviceDescriptions>> de : deviceDescs.entrySet()) {
+            final DeviceId deviceId = de.getKey();
+            final Map<ProviderId, DeviceDescriptions> lDevice = de.getValue();
+
+            synchronized (lDevice) {
+                // latestTimestamp across provider
+                // Note: can be null initially
+                Timestamp localLatest = offline.get(deviceId);
+
+                // handle device Ads
+                for (Entry<ProviderId, DeviceDescriptions> prov : lDevice.entrySet()) {
+                    final ProviderId provId = prov.getKey();
+                    final DeviceDescriptions lDeviceDescs = prov.getValue();
+
+                    final DeviceFragmentId devFragId = new DeviceFragmentId(deviceId, provId);
+
+
+                    Timestamped<DeviceDescription> lProvDevice = lDeviceDescs.getDeviceDesc();
+                    Timestamp advDevTimestamp = devAds.get(devFragId);
+
+                    if (advDevTimestamp == null || lProvDevice.isNewer(advDevTimestamp)) {
+                        // remote does not have it or outdated, suggest
+                        notifyPeer(sender, new InternalDeviceEvent(provId, deviceId, lProvDevice));
+                    } else if (!lProvDevice.timestamp().equals(advDevTimestamp)) {
+                        // local is outdated, request
+                        reqDevices.add(devFragId);
+                    }
+
+                    // handle port Ads
+                    for (Entry<PortNumber, Timestamped<PortDescription>>
+                            pe : lDeviceDescs.getPortDescs().entrySet()) {
+
+                        final PortNumber num = pe.getKey();
+                        final Timestamped<PortDescription> lPort = pe.getValue();
+
+                        final PortFragmentId portFragId = new PortFragmentId(deviceId, provId, num);
+
+                        Timestamp advPortTimestamp = portAds.get(portFragId);
+                        if (advPortTimestamp == null || lPort.isNewer(advPortTimestamp)) {
+                            // remote does not have it or outdated, suggest
+                            notifyPeer(sender, new InternalPortStatusEvent(provId, deviceId, lPort));
+                        } else if (!lPort.timestamp().equals(advPortTimestamp)) {
+                            // local is outdated, request
+                            log.trace("need update {} < {}", lPort.timestamp(), advPortTimestamp);
+                            reqPorts.add(portFragId);
+                        }
+
+                        // remove port Ad already processed
+                        portAds.remove(portFragId);
+                    } // end local port loop
+
+                    // remove device Ad already processed
+                    devAds.remove(devFragId);
+
+                    // find latest and update
+                    final Timestamp providerLatest = lDeviceDescs.getLatestTimestamp();
+                    if (localLatest == null ||
+                        providerLatest.compareTo(localLatest) > 0) {
+                        localLatest = providerLatest;
+                    }
+                } // end local provider loop
+
+                // checking if remote timestamp is more recent.
+                Timestamp rOffline = offlineAds.get(deviceId);
+                if (rOffline != null &&
+                    rOffline.compareTo(localLatest) > 0) {
+                    // remote offline timestamp suggests that the
+                    // device is off-line
+                    markOfflineInternal(deviceId, rOffline);
+                }
+
+                Timestamp lOffline = offline.get(deviceId);
+                if (lOffline != null && rOffline == null) {
+                    // locally offline, but remote is online, suggest offline
+                    notifyPeer(sender, new InternalDeviceOfflineEvent(deviceId, lOffline));
+                }
+
+                // remove device offline Ad already processed
+                offlineAds.remove(deviceId);
+            } // end local device loop
+        } // device lock
+
+        // If there is any Ads left, request them
+        log.trace("Ads left {}, {}", devAds, portAds);
+        reqDevices.addAll(devAds.keySet());
+        reqPorts.addAll(portAds.keySet());
+
+        if (reqDevices.isEmpty() && reqPorts.isEmpty()) {
+            log.trace("Nothing to request to remote peer {}", sender);
+            return;
+        }
+
+        log.info("Need to sync {} {}", reqDevices, reqPorts);
+
+        // 2-way Anti-Entropy for now
+        try {
+            unicastMessage(sender, DEVICE_ADVERTISE, createAdvertisement());
+        } catch (IOException e) {
+            log.error("Failed to send response advertisement to " + sender, e);
+        }
+
+// Sketch of 3-way Anti-Entropy
+//        DeviceAntiEntropyRequest request = new DeviceAntiEntropyRequest(self, reqDevices, reqPorts);
+//        ClusterMessage message = new ClusterMessage(
+//                clusterService.getLocalNode().id(),
+//                GossipDeviceStoreMessageSubjects.DEVICE_REQUEST,
+//                SERIALIZER.encode(request));
+//
+//        try {
+//            clusterCommunicator.unicast(message, advertisement.sender());
+//        } catch (IOException e) {
+//            log.error("Failed to send advertisement reply to "
+//                      + advertisement.sender(), e);
+//        }
     }
     }
 
 
-    private void notifyPeers(InternalPortEvent event) throws IOException {
-        ClusterMessage message = new ClusterMessage(
-                clusterService.getLocalNode().id(),
-                GossipDeviceStoreMessageSubjects.PORT_UPDATE,
-                SERIALIZER.encode(event));
-        clusterCommunicator.broadcast(message);
+    private void notifyDelegateIfNotNull(DeviceEvent event) {
+        if (event != null) {
+            notifyDelegate(event);
+        }
     }
     }
 
 
-    private void notifyPeers(InternalPortStatusEvent event) throws IOException {
-        ClusterMessage message = new ClusterMessage(
-                clusterService.getLocalNode().id(),
-                GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE,
-                SERIALIZER.encode(event));
-        clusterCommunicator.broadcast(message);
+    private final class SendAdvertisementTask implements Runnable {
+
+        @Override
+        public void run() {
+            if (Thread.currentThread().isInterrupted()) {
+                log.info("Interrupted, quitting");
+                return;
+            }
+
+            try {
+                final NodeId self = clusterService.getLocalNode().id();
+                Set<ControllerNode> nodes = clusterService.getNodes();
+
+                ImmutableList<NodeId> nodeIds = FluentIterable.from(nodes)
+                        .transform(toNodeId())
+                        .toList();
+
+                if (nodeIds.size() == 1 && nodeIds.get(0).equals(self)) {
+                    log.info("No other peers in the cluster.");
+                    return;
+                }
+
+                NodeId peer;
+                do {
+                    int idx = RandomUtils.nextInt(0, nodeIds.size());
+                    peer = nodeIds.get(idx);
+                } while (peer.equals(self));
+
+                DeviceAntiEntropyAdvertisement ad = createAdvertisement();
+
+                if (Thread.currentThread().isInterrupted()) {
+                    log.info("Interrupted, quitting");
+                    return;
+                }
+
+                try {
+                    unicastMessage(peer, DEVICE_ADVERTISE, ad);
+                } catch (IOException e) {
+                    log.error("Failed to send anti-entropy advertisement", e);
+                    return;
+                }
+            } catch (Exception e) {
+                // catch all Exception to avoid Scheduled task being suppressed.
+                log.error("Exception thrown while sending advertisement", e);
+            }
+        }
     }
     }
 
 
     private class InternalDeviceEventListener implements ClusterMessageHandler {
     private class InternalDeviceEventListener implements ClusterMessageHandler {
@@ -881,7 +1134,7 @@ public class GossipDeviceStore
             DeviceId deviceId = event.deviceId();
             DeviceId deviceId = event.deviceId();
             Timestamped<DeviceDescription> deviceDescription = event.deviceDescription();
             Timestamped<DeviceDescription> deviceDescription = event.deviceDescription();
 
 
-            createOrUpdateDeviceInternal(providerId, deviceId, deviceDescription);
+            notifyDelegateIfNotNull(createOrUpdateDeviceInternal(providerId, deviceId, deviceDescription));
         }
         }
     }
     }
 
 
@@ -895,7 +1148,7 @@ public class GossipDeviceStore
             DeviceId deviceId = event.deviceId();
             DeviceId deviceId = event.deviceId();
             Timestamp timestamp = event.timestamp();
             Timestamp timestamp = event.timestamp();
 
 
-            markOfflineInternal(deviceId, timestamp);
+            notifyDelegateIfNotNull(markOfflineInternal(deviceId, timestamp));
         }
         }
     }
     }
 
 
@@ -909,7 +1162,7 @@ public class GossipDeviceStore
             DeviceId deviceId = event.deviceId();
             DeviceId deviceId = event.deviceId();
             Timestamp timestamp = event.timestamp();
             Timestamp timestamp = event.timestamp();
 
 
-            removeDeviceInternal(deviceId, timestamp);
+            notifyDelegateIfNotNull(removeDeviceInternal(deviceId, timestamp));
         }
         }
     }
     }
 
 
@@ -924,7 +1177,7 @@ public class GossipDeviceStore
             DeviceId deviceId = event.deviceId();
             DeviceId deviceId = event.deviceId();
             Timestamped<List<PortDescription>> portDescriptions = event.portDescriptions();
             Timestamped<List<PortDescription>> portDescriptions = event.portDescriptions();
 
 
-            updatePortsInternal(providerId, deviceId, portDescriptions);
+            notifyDelegate(updatePortsInternal(providerId, deviceId, portDescriptions));
         }
         }
     }
     }
 
 
@@ -934,12 +1187,24 @@ public class GossipDeviceStore
 
 
             log.info("Received port status update event from peer: {}", message.sender());
             log.info("Received port status update event from peer: {}", message.sender());
             InternalPortStatusEvent event = (InternalPortStatusEvent) SERIALIZER.decode(message.payload());
             InternalPortStatusEvent event = (InternalPortStatusEvent) SERIALIZER.decode(message.payload());
+            log.info("{}", event);
 
 
             ProviderId providerId = event.providerId();
             ProviderId providerId = event.providerId();
             DeviceId deviceId = event.deviceId();
             DeviceId deviceId = event.deviceId();
             Timestamped<PortDescription> portDescription = event.portDescription();
             Timestamped<PortDescription> portDescription = event.portDescription();
 
 
-            updatePortStatusInternal(providerId, deviceId, portDescription);
+            notifyDelegateIfNotNull(updatePortStatusInternal(providerId, deviceId, portDescription));
+        }
+    }
+
+    private final class InternalDeviceAdvertisementListener
+        implements ClusterMessageHandler {
+
+        @Override
+        public void handle(ClusterMessage message) {
+            log.info("Received Device advertisement from peer: {}", message.sender());
+            DeviceAntiEntropyAdvertisement advertisement = SERIALIZER.decode(message.payload());
+            handleAdvertisement(advertisement);
         }
         }
     }
     }
 }
 }

@@ -2,6 +2,7 @@ package org.onlab.onos.store.device.impl;
 
 
 import org.onlab.onos.store.cluster.messaging.MessageSubject;
 import org.onlab.onos.store.cluster.messaging.MessageSubject;
 
 
+// TODO: add prefix to assure uniqueness.
 /**
 /**
  * MessageSubjects used by GossipDeviceStore peer-peer communication.
  * MessageSubjects used by GossipDeviceStore peer-peer communication.
  */
  */
@@ -14,4 +15,8 @@ public final class GossipDeviceStoreMessageSubjects {
     public static final MessageSubject DEVICE_REMOVED = new MessageSubject("peer-device-removed");
     public static final MessageSubject DEVICE_REMOVED = new MessageSubject("peer-device-removed");
     public static final MessageSubject PORT_UPDATE = new MessageSubject("peer-port-update");
     public static final MessageSubject PORT_UPDATE = new MessageSubject("peer-port-update");
     public static final MessageSubject PORT_STATUS_UPDATE = new MessageSubject("peer-port-status-update");
     public static final MessageSubject PORT_STATUS_UPDATE = new MessageSubject("peer-port-status-update");
+
+    public static final MessageSubject DEVICE_ADVERTISE = new MessageSubject("peer-device-advertisements");
+    // to be used with 3-way anti-entropy process
+    public static final MessageSubject DEVICE_REQUEST = new MessageSubject("peer-device-request");
 }
 }

+package org.onlab.onos.store.device.impl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import org.apache.commons.lang3.concurrent.ConcurrentException;
+import org.apache.commons.lang3.concurrent.ConcurrentInitializer;
+import org.onlab.onos.net.device.DeviceDescription;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+// FIXME: consider removing this class
+public final class InitDeviceDescs
+    implements ConcurrentInitializer<DeviceDescriptions> {
+
+    private final Timestamped<DeviceDescription> deviceDesc;
+
+    public InitDeviceDescs(Timestamped<DeviceDescription> deviceDesc) {
+        this.deviceDesc = checkNotNull(deviceDesc);
+    }
+    @Override
+    public DeviceDescriptions get() throws ConcurrentException {
+        return new DeviceDescriptions(deviceDesc);
+    }
+}
\ No newline at end of file

@@ -5,6 +5,8 @@ import org.onlab.onos.net.device.DeviceDescription;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.common.impl.Timestamped;
 import org.onlab.onos.store.common.impl.Timestamped;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a device
  * Information published by GossipDeviceStore to notify peers of a device
  * change event.
  * change event.
@@ -36,6 +38,15 @@ public class InternalDeviceEvent {
         return deviceDescription;
         return deviceDescription;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("deviceDescription", deviceDescription)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     protected InternalDeviceEvent() {
     protected InternalDeviceEvent() {
         this.providerId = null;
         this.providerId = null;

@@ -3,6 +3,8 @@ package org.onlab.onos.store.device.impl;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.store.Timestamp;
 import org.onlab.onos.store.Timestamp;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a device
  * Information published by GossipDeviceStore to notify peers of a device
  * going offline.
  * going offline.
@@ -30,6 +32,14 @@ public class InternalDeviceOfflineEvent {
         return timestamp;
         return timestamp;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("deviceId", deviceId)
+                .add("timestamp", timestamp)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     @SuppressWarnings("unused")
     @SuppressWarnings("unused")
     private InternalDeviceOfflineEvent() {
     private InternalDeviceOfflineEvent() {

@@ -3,6 +3,8 @@ package org.onlab.onos.store.device.impl;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.store.Timestamp;
 import org.onlab.onos.store.Timestamp;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a device
  * Information published by GossipDeviceStore to notify peers of a device
  * being administratively removed.
  * being administratively removed.
@@ -30,6 +32,14 @@ public class InternalDeviceRemovedEvent {
         return timestamp;
         return timestamp;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("deviceId", deviceId)
+                .add("timestamp", timestamp)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     @SuppressWarnings("unused")
     @SuppressWarnings("unused")
     private InternalDeviceRemovedEvent() {
     private InternalDeviceRemovedEvent() {

@@ -7,6 +7,8 @@ import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.common.impl.Timestamped;
 import org.onlab.onos.store.common.impl.Timestamped;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a port
  * Information published by GossipDeviceStore to notify peers of a port
  * change event.
  * change event.
@@ -38,6 +40,15 @@ public class InternalPortEvent {
         return portDescriptions;
         return portDescriptions;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("portDescriptions", portDescriptions)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     protected InternalPortEvent() {
     protected InternalPortEvent() {
         this.providerId = null;
         this.providerId = null;

@@ -5,6 +5,8 @@ import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.common.impl.Timestamped;
 import org.onlab.onos.store.common.impl.Timestamped;
 
 
+import com.google.common.base.MoreObjects;
+
 /**
 /**
  * Information published by GossipDeviceStore to notify peers of a port
  * Information published by GossipDeviceStore to notify peers of a port
  * status change event.
  * status change event.
@@ -36,6 +38,15 @@ public class InternalPortStatusEvent {
         return portDescription;
         return portDescription;
     }
     }
 
 
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("portDescription", portDescription)
+                .toString();
+    }
+
     // for serializer
     // for serializer
     protected InternalPortStatusEvent() {
     protected InternalPortStatusEvent() {
         this.providerId = null;
         this.providerId = null;

@@ -35,6 +35,7 @@ public class InternalPortStatusEventSerializer extends Serializer<InternalPortSt
                                Class<InternalPortStatusEvent> type) {
                                Class<InternalPortStatusEvent> type) {
         ProviderId providerId = (ProviderId) kryo.readClassAndObject(input);
         ProviderId providerId = (ProviderId) kryo.readClassAndObject(input);
         DeviceId deviceId = (DeviceId) kryo.readClassAndObject(input);
         DeviceId deviceId = (DeviceId) kryo.readClassAndObject(input);
+        @SuppressWarnings("unchecked")
         Timestamped<PortDescription> portDescription = (Timestamped<PortDescription>) kryo.readClassAndObject(input);
         Timestamped<PortDescription> portDescription = (Timestamped<PortDescription>) kryo.readClassAndObject(input);
 
 
         return new InternalPortStatusEvent(providerId, deviceId, portDescription);
         return new InternalPortStatusEvent(providerId, deviceId, portDescription);

-package org.onlab.onos.store.device.impl;
-
-import java.util.Objects;
-
-import org.onlab.onos.store.Timestamp;
-
-/**
- * Wrapper class for a entity that is versioned
- * and can either be up or down.
- *
- * @param <T> type of the value.
- */
-public class VersionedValue<T> {
-    private final T entity;
-    private final Timestamp timestamp;
-    private final boolean isUp;
-
-    public VersionedValue(T entity, boolean isUp, Timestamp timestamp) {
-        this.entity = entity;
-        this.isUp = isUp;
-        this.timestamp = timestamp;
-    }
-
-    /**
-     * Returns the value.
-     * @return value.
-     */
-    public T entity() {
-        return entity;
-    }
-
-    /**
-     * Tells whether the entity is up or down.
-     * @return true if up, false otherwise.
-     */
-    public boolean isUp() {
-        return isUp;
-    }
-
-    /**
-     * Returns the timestamp (version) associated with this entity.
-     * @return timestamp.
-     */
-    public Timestamp timestamp() {
-        return timestamp;
-    }
-
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(entity, timestamp, isUp);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (this == obj) {
-            return true;
-        }
-        if (obj == null) {
-            return false;
-        }
-        if (getClass() != obj.getClass()) {
-            return false;
-        }
-        @SuppressWarnings("unchecked")
-        VersionedValue<T> that = (VersionedValue<T>) obj;
-        return Objects.equals(this.entity, that.entity) &&
-                Objects.equals(this.timestamp, that.timestamp) &&
-                Objects.equals(this.isUp, that.isUp);
-    }
-
-    // Default constructor for serializer
-    protected VersionedValue() {
-        this.entity = null;
-        this.isUp = false;
-        this.timestamp = null;
-    }
-}

+package org.onlab.onos.store.device.impl.peermsg;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Map;
+
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.store.Timestamp;
+
+
+/**
+ * Device Advertisement message.
+ */
+public class DeviceAntiEntropyAdvertisement {
+
+    private final NodeId sender;
+    private final Map<DeviceFragmentId, Timestamp> deviceFingerPrints;
+    private final Map<PortFragmentId, Timestamp> portFingerPrints;
+    private final Map<DeviceId, Timestamp> offline;
+
+
+    public DeviceAntiEntropyAdvertisement(NodeId sender,
+                Map<DeviceFragmentId, Timestamp> devices,
+                Map<PortFragmentId, Timestamp> ports,
+                Map<DeviceId, Timestamp> offline) {
+        this.sender = checkNotNull(sender);
+        this.deviceFingerPrints = checkNotNull(devices);
+        this.portFingerPrints = checkNotNull(ports);
+        this.offline = checkNotNull(offline);
+    }
+
+    public NodeId sender() {
+        return sender;
+    }
+
+    public Map<DeviceFragmentId, Timestamp> deviceFingerPrints() {
+        return deviceFingerPrints;
+    }
+
+    public Map<PortFragmentId, Timestamp> ports() {
+        return portFingerPrints;
+    }
+
+    public Map<DeviceId, Timestamp> offline() {
+        return offline;
+    }
+
+    // For serializer
+    @SuppressWarnings("unused")
+    private DeviceAntiEntropyAdvertisement() {
+        this.sender = null;
+        this.deviceFingerPrints = null;
+        this.portFingerPrints = null;
+        this.offline = null;
+    }
+}

+package org.onlab.onos.store.device.impl.peermsg;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Collection;
+
+import org.onlab.onos.cluster.NodeId;
+
+/**
+ * Message to request for other peers information.
+ */
+public class DeviceAntiEntropyRequest {
+
+    private final NodeId sender;
+    private final Collection<DeviceFragmentId> devices;
+    private final Collection<PortFragmentId> ports;
+
+    public DeviceAntiEntropyRequest(NodeId sender,
+                                   Collection<DeviceFragmentId> devices,
+                                   Collection<PortFragmentId> ports) {
+
+        this.sender = checkNotNull(sender);
+        this.devices = checkNotNull(devices);
+        this.ports = checkNotNull(ports);
+    }
+
+    public NodeId sender() {
+        return sender;
+    }
+
+    public Collection<DeviceFragmentId> devices() {
+        return devices;
+    }
+
+    public Collection<PortFragmentId> ports() {
+        return ports;
+    }
+
+    // For serializer
+    @SuppressWarnings("unused")
+    private DeviceAntiEntropyRequest() {
+        this.sender = null;
+        this.devices = null;
+        this.ports = null;
+    }
+}

+package org.onlab.onos.store.device.impl.peermsg;
+
+import java.util.Objects;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.provider.ProviderId;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Identifier for DeviceDesctiption from a Provider.
+ */
+public final class DeviceFragmentId {
+    public final ProviderId providerId;
+    public final DeviceId deviceId;
+
+    public DeviceFragmentId(DeviceId deviceId, ProviderId providerId) {
+        this.providerId = providerId;
+        this.deviceId = deviceId;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(providerId, deviceId);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof DeviceFragmentId)) {
+            return false;
+        }
+        DeviceFragmentId that = (DeviceFragmentId) obj;
+        return Objects.equals(this.deviceId, that.deviceId) &&
+               Objects.equals(this.providerId, that.providerId);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .toString();
+    }
+
+    // for serializer
+    @SuppressWarnings("unused")
+    private DeviceFragmentId() {
+        this.providerId = null;
+        this.deviceId = null;
+    }
+}
\ No newline at end of file

+package org.onlab.onos.store.device.impl.peermsg;
+
+import java.util.Objects;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.provider.ProviderId;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Identifier for PortDescription from a Provider.
+ */
+public final class PortFragmentId {
+    public final ProviderId providerId;
+    public final DeviceId deviceId;
+    public final PortNumber portNumber;
+
+    public PortFragmentId(DeviceId deviceId, ProviderId providerId,
+                          PortNumber portNumber) {
+        this.providerId = providerId;
+        this.deviceId = deviceId;
+        this.portNumber = portNumber;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(providerId, deviceId, portNumber);
+    };
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof PortFragmentId)) {
+            return false;
+        }
+        PortFragmentId that = (PortFragmentId) obj;
+        return Objects.equals(this.deviceId, that.deviceId) &&
+               Objects.equals(this.portNumber, that.portNumber) &&
+               Objects.equals(this.providerId, that.providerId);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("deviceId", deviceId)
+                .add("portNumber", portNumber)
+                .toString();
+    }
+
+    // for serializer
+    @SuppressWarnings("unused")
+    private PortFragmentId() {
+        this.providerId = null;
+        this.deviceId = null;
+        this.portNumber = null;
+    }
+}
\ No newline at end of file

+/**
+ * Structure and utilities used for inter-Node messaging.
+ */
+package org.onlab.onos.store.device.impl.peermsg;

@@ -43,8 +43,8 @@ public class DistributedFlowRuleStore
     private final Multimap<DeviceId, FlowEntry> flowEntries =
     private final Multimap<DeviceId, FlowEntry> flowEntries =
             ArrayListMultimap.<DeviceId, FlowEntry>create();
             ArrayListMultimap.<DeviceId, FlowEntry>create();
 
 
-    private final Multimap<ApplicationId, FlowRule> flowEntriesById =
-            ArrayListMultimap.<ApplicationId, FlowRule>create();
+    private final Multimap<Short, FlowRule> flowEntriesById =
+            ArrayListMultimap.<Short, FlowRule>create();
 
 
     @Activate
     @Activate
     public void activate() {
     public void activate() {
@@ -83,7 +83,7 @@ public class DistributedFlowRuleStore
 
 
     @Override
     @Override
     public synchronized Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId) {
     public synchronized Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId) {
-        Collection<FlowRule> rules = flowEntriesById.get(appId);
+        Collection<FlowRule> rules = flowEntriesById.get(appId.id());
         if (rules == null) {
         if (rules == null) {
             return Collections.emptyList();
             return Collections.emptyList();
         }
         }

 package org.onlab.onos.store.host.impl;
 package org.onlab.onos.store.host.impl;
 
 
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_ADDED;
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_MOVED;
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_REMOVED;
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_UPDATED;
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.net.Annotations;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.DefaultHost;
 import org.onlab.onos.net.DefaultHost;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.HostId;
 import org.onlab.onos.net.HostId;
+import org.onlab.onos.net.HostLocation;
 import org.onlab.onos.net.host.HostDescription;
 import org.onlab.onos.net.host.HostDescription;
 import org.onlab.onos.net.host.HostEvent;
 import org.onlab.onos.net.host.HostEvent;
 import org.onlab.onos.net.host.HostStore;
 import org.onlab.onos.net.host.HostStore;
@@ -33,10 +27,13 @@ import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.onlab.packet.VlanId;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Sets;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.onlab.onos.net.host.HostEvent.Type.*;
+import static org.slf4j.LoggerFactory.getLogger;
 
 
 /**
 /**
  * Manages inventory of end-station hosts using trivial in-memory
  * Manages inventory of end-station hosts using trivial in-memory
@@ -46,13 +43,13 @@ import com.google.common.collect.Sets;
 @Component(immediate = true)
 @Component(immediate = true)
 @Service
 @Service
 public class DistributedHostStore
 public class DistributedHostStore
-extends AbstractStore<HostEvent, HostStoreDelegate>
-implements HostStore {
+        extends AbstractStore<HostEvent, HostStoreDelegate>
+        implements HostStore {
 
 
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
 
 
     // Host inventory
     // Host inventory
-    private final Map<HostId, Host> hosts = new ConcurrentHashMap<>();
+    private final Map<HostId, StoredHost> hosts = new ConcurrentHashMap<>(2000000, 0.75f, 16);
 
 
     // Hosts tracked by their location
     // Hosts tracked by their location
     private final Multimap<ConnectPoint, Host> locations = HashMultimap.create();
     private final Multimap<ConnectPoint, Host> locations = HashMultimap.create();
@@ -72,8 +69,8 @@ implements HostStore {
 
 
     @Override
     @Override
     public HostEvent createOrUpdateHost(ProviderId providerId, HostId hostId,
     public HostEvent createOrUpdateHost(ProviderId providerId, HostId hostId,
-            HostDescription hostDescription) {
-        Host host = hosts.get(hostId);
+                                        HostDescription hostDescription) {
+        StoredHost host = hosts.get(hostId);
         if (host == null) {
         if (host == null) {
             return createHost(providerId, hostId, hostDescription);
             return createHost(providerId, hostId, hostDescription);
         }
         }
@@ -82,12 +79,12 @@ implements HostStore {
 
 
     // creates a new host and sends HOST_ADDED
     // creates a new host and sends HOST_ADDED
     private HostEvent createHost(ProviderId providerId, HostId hostId,
     private HostEvent createHost(ProviderId providerId, HostId hostId,
-            HostDescription descr) {
-        DefaultHost newhost = new DefaultHost(providerId, hostId,
-                descr.hwAddress(),
-                descr.vlan(),
-                descr.location(),
-                descr.ipAddresses());
+                                 HostDescription descr) {
+        StoredHost newhost = new StoredHost(providerId, hostId,
+                                            descr.hwAddress(),
+                                            descr.vlan(),
+                                            descr.location(),
+                                            ImmutableSet.of(descr.ipAddress()));
         synchronized (this) {
         synchronized (this) {
             hosts.put(hostId, newhost);
             hosts.put(hostId, newhost);
             locations.put(descr.location(), newhost);
             locations.put(descr.location(), newhost);
@@ -96,28 +93,24 @@ implements HostStore {
     }
     }
 
 
     // checks for type of update to host, sends appropriate event
     // checks for type of update to host, sends appropriate event
-    private HostEvent updateHost(ProviderId providerId, Host host,
-            HostDescription descr) {
-        DefaultHost updated;
+    private HostEvent updateHost(ProviderId providerId, StoredHost host,
+                                 HostDescription descr) {
         HostEvent event;
         HostEvent event;
         if (!host.location().equals(descr.location())) {
         if (!host.location().equals(descr.location())) {
-            updated = new DefaultHost(providerId, host.id(),
-                    host.mac(),
-                    host.vlan(),
-                    descr.location(),
-                    host.ipAddresses());
-            event = new HostEvent(HOST_MOVED, updated);
-
-        } else if (!(host.ipAddresses().equals(descr.ipAddresses()))) {
-            updated = new DefaultHost(providerId, host.id(),
-                    host.mac(),
-                    host.vlan(),
-                    descr.location(),
-                    descr.ipAddresses());
-            event = new HostEvent(HOST_UPDATED, updated);
-        } else {
+            host.setLocation(descr.location());
+            return new HostEvent(HOST_MOVED, host);
+        }
+
+        if (host.ipAddresses().contains(descr.ipAddress())) {
             return null;
             return null;
         }
         }
+
+        Set<IpPrefix> addresses = new HashSet<>(host.ipAddresses());
+        addresses.add(descr.ipAddress());
+        StoredHost updated = new StoredHost(providerId, host.id(),
+                                            host.mac(), host.vlan(),
+                                            descr.location(), addresses);
+        event = new HostEvent(HOST_UPDATED, updated);
         synchronized (this) {
         synchronized (this) {
             hosts.put(host.id(), updated);
             hosts.put(host.id(), updated);
             locations.remove(host.location(), host);
             locations.remove(host.location(), host);
@@ -145,7 +138,7 @@ implements HostStore {
 
 
     @Override
     @Override
     public Iterable<Host> getHosts() {
     public Iterable<Host> getHosts() {
-        return Collections.unmodifiableSet(new HashSet<>(hosts.values()));
+        return ImmutableSet.<Host>copyOf(hosts.values());
     }
     }
 
 
     @Override
     @Override
@@ -275,4 +268,35 @@ implements HostStore {
         return addresses;
         return addresses;
     }
     }
 
 
+    // Auxiliary extension to allow location to mutate.
+    private class StoredHost extends DefaultHost {
+        private HostLocation location;
+
+        /**
+         * Creates an end-station host using the supplied information.
+         *
+         * @param providerId  provider identity
+         * @param id          host identifier
+         * @param mac         host MAC address
+         * @param vlan        host VLAN identifier
+         * @param location    host location
+         * @param ips         host IP addresses
+         * @param annotations optional key/value annotations
+         */
+        public StoredHost(ProviderId providerId, HostId id,
+                          MacAddress mac, VlanId vlan, HostLocation location,
+                          Set<IpPrefix> ips, Annotations... annotations) {
+            super(providerId, id, mac, vlan, location, ips, annotations);
+            this.location = location;
+        }
+
+        void setLocation(HostLocation location) {
+            this.location = location;
+        }
+
+        @Override
+        public HostLocation location() {
+            return location;
+        }
+    }
 }
 }

+package org.onlab.onos.store.link.impl;
+
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.SetMultimap;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.commons.lang3.concurrent.ConcurrentUtils;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.cluster.ClusterService;
+import org.onlab.onos.cluster.ControllerNode;
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.net.AnnotationsUtil;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DefaultAnnotations;
+import org.onlab.onos.net.DefaultLink;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.SparseAnnotations;
+import org.onlab.onos.net.Link.Type;
+import org.onlab.onos.net.LinkKey;
+import org.onlab.onos.net.Provided;
+import org.onlab.onos.net.link.DefaultLinkDescription;
+import org.onlab.onos.net.link.LinkDescription;
+import org.onlab.onos.net.link.LinkEvent;
+import org.onlab.onos.net.link.LinkStore;
+import org.onlab.onos.net.link.LinkStoreDelegate;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.AbstractStore;
+import org.onlab.onos.store.ClockService;
+import org.onlab.onos.store.Timestamp;
+import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
+import org.onlab.onos.store.cluster.messaging.ClusterMessage;
+import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
+import org.onlab.onos.store.common.impl.Timestamped;
+import org.onlab.onos.store.serializers.DistributedStoreSerializers;
+import org.onlab.onos.store.serializers.KryoSerializer;
+import org.onlab.util.KryoPool;
+import org.onlab.util.NewConcurrentHashMap;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
+import static java.util.concurrent.Executors.newSingleThreadScheduledExecutor;
+import static org.onlab.onos.cluster.ControllerNodeToNodeId.toNodeId;
+import static org.onlab.onos.net.DefaultAnnotations.union;
+import static org.onlab.onos.net.DefaultAnnotations.merge;
+import static org.onlab.onos.net.Link.Type.DIRECT;
+import static org.onlab.onos.net.Link.Type.INDIRECT;
+import static org.onlab.onos.net.link.LinkEvent.Type.*;
+import static org.onlab.util.Tools.namedThreads;
+import static org.slf4j.LoggerFactory.getLogger;
+import static com.google.common.collect.Multimaps.synchronizedSetMultimap;
+import static com.google.common.base.Predicates.notNull;
+
+/**
+ * Manages inventory of infrastructure links in distributed data store
+ * that uses optimistic replication and gossip based techniques.
+ */
+@Component(immediate = true)
+@Service
+public class GossipLinkStore
+        extends AbstractStore<LinkEvent, LinkStoreDelegate>
+        implements LinkStore {
+
+    private final Logger log = getLogger(getClass());
+
+    // Link inventory
+    private final ConcurrentMap<LinkKey, ConcurrentMap<ProviderId, Timestamped<LinkDescription>>> linkDescs =
+        new ConcurrentHashMap<>();
+
+    // Link instance cache
+    private final ConcurrentMap<LinkKey, Link> links = new ConcurrentHashMap<>();
+
+    // Egress and ingress link sets
+    private final SetMultimap<DeviceId, LinkKey> srcLinks = createSynchronizedHashMultiMap();
+    private final SetMultimap<DeviceId, LinkKey> dstLinks = createSynchronizedHashMultiMap();
+
+    // Remove links
+    private final Map<LinkKey, Timestamp> removedLinks = Maps.newHashMap();
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClockService clockService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterCommunicationService clusterCommunicator;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterService clusterService;
+
+    private static final KryoSerializer SERIALIZER = new KryoSerializer() {
+        @Override
+        protected void setupKryoPool() {
+            serializerPool = KryoPool.newBuilder()
+                    .register(DistributedStoreSerializers.COMMON)
+                    .register(InternalLinkEvent.class)
+                    .register(InternalLinkRemovedEvent.class)
+                    .register(LinkAntiEntropyAdvertisement.class)
+                    .register(LinkFragmentId.class)
+                    .build()
+                    .populate(1);
+        }
+    };
+
+    private ScheduledExecutorService executor;
+
+    @Activate
+    public void activate() {
+
+        clusterCommunicator.addSubscriber(
+                GossipLinkStoreMessageSubjects.LINK_UPDATE,
+                new InternalLinkEventListener());
+        clusterCommunicator.addSubscriber(
+                GossipLinkStoreMessageSubjects.LINK_REMOVED,
+                new InternalLinkRemovedEventListener());
+        clusterCommunicator.addSubscriber(
+                GossipLinkStoreMessageSubjects.LINK_ANTI_ENTROPY_ADVERTISEMENT,
+                new InternalLinkAntiEntropyAdvertisementListener());
+
+        executor =
+                newSingleThreadScheduledExecutor(namedThreads("link-anti-entropy-%d"));
+
+        // TODO: Make these configurable
+        long initialDelaySec = 5;
+        long periodSec = 5;
+        // start anti-entropy thread
+        executor.scheduleAtFixedRate(new SendAdvertisementTask(),
+                    initialDelaySec, periodSec, TimeUnit.SECONDS);
+
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        linkDescs.clear();
+        links.clear();
+        srcLinks.clear();
+        dstLinks.clear();
+        log.info("Stopped");
+    }
+
+    @Override
+    public int getLinkCount() {
+        return links.size();
+    }
+
+    @Override
+    public Iterable<Link> getLinks() {
+        return Collections.unmodifiableCollection(links.values());
+    }
+
+    @Override
+    public Set<Link> getDeviceEgressLinks(DeviceId deviceId) {
+        // lock for iteration
+        synchronized (srcLinks) {
+            return FluentIterable.from(srcLinks.get(deviceId))
+            .transform(lookupLink())
+            .filter(notNull())
+            .toSet();
+        }
+    }
+
+    @Override
+    public Set<Link> getDeviceIngressLinks(DeviceId deviceId) {
+        // lock for iteration
+        synchronized (dstLinks) {
+            return FluentIterable.from(dstLinks.get(deviceId))
+            .transform(lookupLink())
+            .filter(notNull())
+            .toSet();
+        }
+    }
+
+    @Override
+    public Link getLink(ConnectPoint src, ConnectPoint dst) {
+        return links.get(new LinkKey(src, dst));
+    }
+
+    @Override
+    public Set<Link> getEgressLinks(ConnectPoint src) {
+        Set<Link> egress = new HashSet<>();
+        for (LinkKey linkKey : srcLinks.get(src.deviceId())) {
+            if (linkKey.src().equals(src)) {
+                egress.add(links.get(linkKey));
+            }
+        }
+        return egress;
+    }
+
+    @Override
+    public Set<Link> getIngressLinks(ConnectPoint dst) {
+        Set<Link> ingress = new HashSet<>();
+        for (LinkKey linkKey : dstLinks.get(dst.deviceId())) {
+            if (linkKey.dst().equals(dst)) {
+                ingress.add(links.get(linkKey));
+            }
+        }
+        return ingress;
+    }
+
+    @Override
+    public LinkEvent createOrUpdateLink(ProviderId providerId,
+                                        LinkDescription linkDescription) {
+
+        DeviceId dstDeviceId = linkDescription.dst().deviceId();
+        Timestamp newTimestamp = clockService.getTimestamp(dstDeviceId);
+
+        final Timestamped<LinkDescription> deltaDesc = new Timestamped<>(linkDescription, newTimestamp);
+
+        LinkEvent event = createOrUpdateLinkInternal(providerId, deltaDesc);
+
+        if (event != null) {
+            log.info("Notifying peers of a link update topology event from providerId: "
+                    + "{}  between src: {} and dst: {}",
+                    providerId, linkDescription.src(), linkDescription.dst());
+            try {
+                notifyPeers(new InternalLinkEvent(providerId, deltaDesc));
+            } catch (IOException e) {
+                log.info("Failed to notify peers of a link update topology event from providerId: "
+                        + "{}  between src: {} and dst: {}",
+                        providerId, linkDescription.src(), linkDescription.dst());
+            }
+        }
+        return event;
+    }
+
+    private LinkEvent createOrUpdateLinkInternal(
+            ProviderId providerId,
+            Timestamped<LinkDescription> linkDescription) {
+
+        LinkKey key = new LinkKey(linkDescription.value().src(), linkDescription.value().dst());
+        ConcurrentMap<ProviderId, Timestamped<LinkDescription>> descs = getLinkDescriptions(key);
+
+        synchronized (descs) {
+            // if the link was previously removed, we should proceed if and
+            // only if this request is more recent.
+            Timestamp linkRemovedTimestamp = removedLinks.get(key);
+            if (linkRemovedTimestamp != null) {
+                if (linkDescription.isNewer(linkRemovedTimestamp)) {
+                    removedLinks.remove(key);
+                } else {
+                    return null;
+                }
+            }
+
+            final Link oldLink = links.get(key);
+            // update description
+            createOrUpdateLinkDescription(descs, providerId, linkDescription);
+            final Link newLink = composeLink(descs);
+            if (oldLink == null) {
+                return createLink(key, newLink);
+            }
+            return updateLink(key, oldLink, newLink);
+        }
+    }
+
+    // Guarded by linkDescs value (=locking each Link)
+    private Timestamped<LinkDescription> createOrUpdateLinkDescription(
+            ConcurrentMap<ProviderId, Timestamped<LinkDescription>> existingLinkDescriptions,
+            ProviderId providerId,
+            Timestamped<LinkDescription> linkDescription) {
+
+        // merge existing attributes and merge
+        Timestamped<LinkDescription> existingLinkDescription = existingLinkDescriptions.get(providerId);
+        if (existingLinkDescription != null && existingLinkDescription.isNewer(linkDescription)) {
+            return null;
+        }
+        Timestamped<LinkDescription> newLinkDescription = linkDescription;
+        if (existingLinkDescription != null) {
+            SparseAnnotations merged = union(existingLinkDescription.value().annotations(),
+                    linkDescription.value().annotations());
+            newLinkDescription = new Timestamped<LinkDescription>(
+                    new DefaultLinkDescription(
+                        linkDescription.value().src(),
+                        linkDescription.value().dst(),
+                        linkDescription.value().type(), merged),
+                    linkDescription.timestamp());
+        }
+        return existingLinkDescriptions.put(providerId, newLinkDescription);
+    }
+
+    // Creates and stores the link and returns the appropriate event.
+    // Guarded by linkDescs value (=locking each Link)
+    private LinkEvent createLink(LinkKey key, Link newLink) {
+
+        if (newLink.providerId().isAncillary()) {
+            // TODO: revisit ancillary only Link handling
+
+            // currently treating ancillary only as down (not visible outside)
+            return null;
+        }
+
+        links.put(key, newLink);
+        srcLinks.put(newLink.src().deviceId(), key);
+        dstLinks.put(newLink.dst().deviceId(), key);
+        return new LinkEvent(LINK_ADDED, newLink);
+    }
+
+    // Updates, if necessary the specified link and returns the appropriate event.
+    // Guarded by linkDescs value (=locking each Link)
+    private LinkEvent updateLink(LinkKey key, Link oldLink, Link newLink) {
+
+        if (newLink.providerId().isAncillary()) {
+            // TODO: revisit ancillary only Link handling
+
+            // currently treating ancillary only as down (not visible outside)
+            return null;
+        }
+
+        if ((oldLink.type() == INDIRECT && newLink.type() == DIRECT) ||
+            !AnnotationsUtil.isEqual(oldLink.annotations(), newLink.annotations())) {
+
+            links.put(key, newLink);
+            // strictly speaking following can be ommitted
+            srcLinks.put(oldLink.src().deviceId(), key);
+            dstLinks.put(oldLink.dst().deviceId(), key);
+            return new LinkEvent(LINK_UPDATED, newLink);
+        }
+        return null;
+    }
+
+    @Override
+    public LinkEvent removeLink(ConnectPoint src, ConnectPoint dst) {
+        final LinkKey key = new LinkKey(src, dst);
+
+        DeviceId dstDeviceId = dst.deviceId();
+        Timestamp timestamp = clockService.getTimestamp(dstDeviceId);
+
+        LinkEvent event = removeLinkInternal(key, timestamp);
+
+        if (event != null) {
+            log.info("Notifying peers of a link removed topology event for a link "
+                    + "between src: {} and dst: {}", src, dst);
+            try {
+                notifyPeers(new InternalLinkRemovedEvent(key, timestamp));
+            } catch (IOException e) {
+                log.error("Failed to notify peers of a link removed topology event for a link "
+                        + "between src: {} and dst: {}", src, dst);
+            }
+        }
+        return event;
+    }
+
+    private LinkEvent removeLinkInternal(LinkKey key, Timestamp timestamp) {
+        ConcurrentMap<ProviderId, Timestamped<LinkDescription>> linkDescriptions =
+                getLinkDescriptions(key);
+        synchronized (linkDescriptions) {
+            // accept removal request if given timestamp is newer than
+            // the latest Timestamp from Primary provider
+            ProviderId primaryProviderId = pickPrimaryProviderId(linkDescriptions);
+            if (linkDescriptions.get(primaryProviderId).isNewer(timestamp)) {
+                return null;
+            }
+            removedLinks.put(key, timestamp);
+            Link link = links.remove(key);
+            linkDescriptions.clear();
+            if (link != null) {
+                srcLinks.remove(link.src().deviceId(), key);
+                dstLinks.remove(link.dst().deviceId(), key);
+                return new LinkEvent(LINK_REMOVED, link);
+            }
+            return null;
+        }
+    }
+
+    private static <K, V> SetMultimap<K, V> createSynchronizedHashMultiMap() {
+        return synchronizedSetMultimap(HashMultimap.<K, V>create());
+    }
+
+    /**
+     * @return primary ProviderID, or randomly chosen one if none exists
+     */
+    private ProviderId pickPrimaryProviderId(
+            ConcurrentMap<ProviderId, Timestamped<LinkDescription>> providerDescs) {
+
+        ProviderId fallBackPrimary = null;
+        for (Entry<ProviderId, Timestamped<LinkDescription>> e : providerDescs.entrySet()) {
+            if (!e.getKey().isAncillary()) {
+                return e.getKey();
+            } else if (fallBackPrimary == null) {
+                // pick randomly as a fallback in case there is no primary
+                fallBackPrimary = e.getKey();
+            }
+        }
+        return fallBackPrimary;
+    }
+
+    private Link composeLink(ConcurrentMap<ProviderId, Timestamped<LinkDescription>> linkDescriptions) {
+        ProviderId primaryProviderId = pickPrimaryProviderId(linkDescriptions);
+        Timestamped<LinkDescription> base = linkDescriptions.get(primaryProviderId);
+
+        ConnectPoint src = base.value().src();
+        ConnectPoint dst = base.value().dst();
+        Type type = base.value().type();
+        DefaultAnnotations annotations = DefaultAnnotations.builder().build();
+        annotations = merge(annotations, base.value().annotations());
+
+        for (Entry<ProviderId, Timestamped<LinkDescription>> e : linkDescriptions.entrySet()) {
+            if (primaryProviderId.equals(e.getKey())) {
+                continue;
+            }
+
+            // TODO: should keep track of Description timestamp
+            // and only merge conflicting keys when timestamp is newer
+            // Currently assuming there will never be a key conflict between
+            // providers
+
+            // annotation merging. not so efficient, should revisit later
+            annotations = merge(annotations, e.getValue().value().annotations());
+        }
+
+        return new DefaultLink(primaryProviderId , src, dst, type, annotations);
+    }
+
+    private ConcurrentMap<ProviderId, Timestamped<LinkDescription>> getLinkDescriptions(LinkKey key) {
+        return ConcurrentUtils.createIfAbsentUnchecked(linkDescs, key,
+                NewConcurrentHashMap.<ProviderId, Timestamped<LinkDescription>>ifNeeded());
+    }
+
+    private Timestamped<LinkDescription> getLinkDescription(LinkKey key, ProviderId providerId) {
+        return getLinkDescriptions(key).get(providerId);
+    }
+
+    private final Function<LinkKey, Link> lookupLink = new LookupLink();
+    private Function<LinkKey, Link> lookupLink() {
+        return lookupLink;
+    }
+
+    private final class LookupLink implements Function<LinkKey, Link> {
+        @Override
+        public Link apply(LinkKey input) {
+            return links.get(input);
+        }
+    }
+
+    private static final Predicate<Provided> IS_PRIMARY = new IsPrimary();
+    private static final Predicate<Provided> isPrimary() {
+        return IS_PRIMARY;
+    }
+
+    private static final class IsPrimary implements Predicate<Provided> {
+
+        @Override
+        public boolean apply(Provided input) {
+            return !input.providerId().isAncillary();
+        }
+    }
+
+    private void notifyDelegateIfNotNull(LinkEvent event) {
+        if (event != null) {
+            notifyDelegate(event);
+        }
+    }
+
+    // TODO: should we be throwing exception?
+    private void broadcastMessage(MessageSubject subject, Object event) throws IOException {
+        ClusterMessage message = new ClusterMessage(
+                clusterService.getLocalNode().id(),
+                subject,
+                SERIALIZER.encode(event));
+        clusterCommunicator.broadcast(message);
+    }
+
+    // TODO: should we be throwing exception?
+    private void unicastMessage(NodeId recipient, MessageSubject subject, Object event) {
+        try {
+            ClusterMessage message = new ClusterMessage(
+                    clusterService.getLocalNode().id(),
+                    subject,
+                    SERIALIZER.encode(event));
+            clusterCommunicator.unicast(message, recipient);
+        } catch (IOException e) {
+            log.error("Failed to send a {} message to {}", subject.value(), recipient);
+        }
+    }
+
+    private void notifyPeers(InternalLinkEvent event) throws IOException {
+        broadcastMessage(GossipLinkStoreMessageSubjects.LINK_UPDATE, event);
+    }
+
+    private void notifyPeers(InternalLinkRemovedEvent event) throws IOException {
+        broadcastMessage(GossipLinkStoreMessageSubjects.LINK_REMOVED, event);
+    }
+
+    private void notifyPeer(NodeId peer, InternalLinkEvent event) {
+        unicastMessage(peer, GossipLinkStoreMessageSubjects.LINK_UPDATE, event);
+    }
+
+    private void notifyPeer(NodeId peer, InternalLinkRemovedEvent event) {
+        unicastMessage(peer, GossipLinkStoreMessageSubjects.LINK_REMOVED, event);
+    }
+
+    private final class SendAdvertisementTask implements Runnable {
+
+        @Override
+        public void run() {
+            if (Thread.currentThread().isInterrupted()) {
+                log.info("Interrupted, quitting");
+                return;
+            }
+
+            try {
+                final NodeId self = clusterService.getLocalNode().id();
+                Set<ControllerNode> nodes = clusterService.getNodes();
+
+                ImmutableList<NodeId> nodeIds = FluentIterable.from(nodes)
+                        .transform(toNodeId())
+                        .toList();
+
+                if (nodeIds.size() == 1 && nodeIds.get(0).equals(self)) {
+                    log.info("No other peers in the cluster.");
+                    return;
+                }
+
+                NodeId peer;
+                do {
+                    int idx = RandomUtils.nextInt(0, nodeIds.size());
+                    peer = nodeIds.get(idx);
+                } while (peer.equals(self));
+
+                LinkAntiEntropyAdvertisement ad = createAdvertisement();
+
+                if (Thread.currentThread().isInterrupted()) {
+                    log.info("Interrupted, quitting");
+                    return;
+                }
+
+                try {
+                    unicastMessage(peer, GossipLinkStoreMessageSubjects.LINK_ANTI_ENTROPY_ADVERTISEMENT, ad);
+                } catch (Exception e) {
+                    log.error("Failed to send anti-entropy advertisement", e);
+                    return;
+                }
+            } catch (Exception e) {
+                // catch all Exception to avoid Scheduled task being suppressed.
+                log.error("Exception thrown while sending advertisement", e);
+            }
+        }
+    }
+
+    private LinkAntiEntropyAdvertisement createAdvertisement() {
+        final NodeId self = clusterService.getLocalNode().id();
+
+        Map<LinkFragmentId, Timestamp> linkTimestamps = new HashMap<>(linkDescs.size());
+        Map<LinkKey, Timestamp> linkTombstones = new HashMap<>(removedLinks.size());
+
+        for (Entry<LinkKey, ConcurrentMap<ProviderId, Timestamped<LinkDescription>>>
+            provs : linkDescs.entrySet()) {
+
+            final LinkKey linkKey = provs.getKey();
+            final ConcurrentMap<ProviderId, Timestamped<LinkDescription>> linkDesc = provs.getValue();
+            synchronized (linkDesc) {
+                for (Map.Entry<ProviderId, Timestamped<LinkDescription>> e : linkDesc.entrySet()) {
+                    linkTimestamps.put(new LinkFragmentId(linkKey, e.getKey()), e.getValue().timestamp());
+                }
+            }
+        }
+
+        linkTombstones.putAll(removedLinks);
+
+        return new LinkAntiEntropyAdvertisement(self, linkTimestamps, linkTombstones);
+    }
+
+    private void handleAntiEntropyAdvertisement(LinkAntiEntropyAdvertisement advertisement) {
+
+        NodeId peer = advertisement.sender();
+
+        Map<LinkFragmentId, Timestamp> linkTimestamps = advertisement.linkTimestamps();
+        Map<LinkKey, Timestamp> linkTombstones = advertisement.linkTombstones();
+        for (Map.Entry<LinkFragmentId, Timestamp> entry : linkTimestamps.entrySet()) {
+            LinkFragmentId linkFragmentId = entry.getKey();
+            Timestamp peerTimestamp = entry.getValue();
+
+            LinkKey key = linkFragmentId.linkKey();
+            ProviderId providerId = linkFragmentId.providerId();
+
+            Timestamped<LinkDescription> linkDescription = getLinkDescription(key, providerId);
+            if (linkDescription.isNewer(peerTimestamp)) {
+                // I have more recent link description. update peer.
+                notifyPeer(peer, new InternalLinkEvent(providerId, linkDescription));
+            }
+            // else TODO: Peer has more recent link description. request it.
+
+            Timestamp linkRemovedTimestamp = removedLinks.get(key);
+            if (linkRemovedTimestamp != null && linkRemovedTimestamp.compareTo(peerTimestamp) > 0) {
+                // peer has a zombie link. update peer.
+                notifyPeer(peer, new InternalLinkRemovedEvent(key, linkRemovedTimestamp));
+            }
+        }
+
+        for (Map.Entry<LinkKey, Timestamp> entry : linkTombstones.entrySet()) {
+            LinkKey key = entry.getKey();
+            Timestamp peerTimestamp = entry.getValue();
+
+            ProviderId primaryProviderId = pickPrimaryProviderId(getLinkDescriptions(key));
+            if (primaryProviderId != null) {
+                if (!getLinkDescription(key, primaryProviderId).isNewer(peerTimestamp)) {
+                    notifyDelegateIfNotNull(removeLinkInternal(key, peerTimestamp));
+                }
+            }
+        }
+    }
+
+    private class InternalLinkEventListener implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.info("Received link event from peer: {}", message.sender());
+            InternalLinkEvent event = (InternalLinkEvent) SERIALIZER.decode(message.payload());
+
+            ProviderId providerId = event.providerId();
+            Timestamped<LinkDescription> linkDescription = event.linkDescription();
+
+            notifyDelegateIfNotNull(createOrUpdateLinkInternal(providerId, linkDescription));
+        }
+    }
+
+    private class InternalLinkRemovedEventListener implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.info("Received link removed event from peer: {}", message.sender());
+            InternalLinkRemovedEvent event = (InternalLinkRemovedEvent) SERIALIZER.decode(message.payload());
+
+            LinkKey linkKey = event.linkKey();
+            Timestamp timestamp = event.timestamp();
+
+            notifyDelegateIfNotNull(removeLinkInternal(linkKey, timestamp));
+        }
+    }
+
+    private final class InternalLinkAntiEntropyAdvertisementListener implements ClusterMessageHandler {
+
+        @Override
+        public void handle(ClusterMessage message) {
+            log.info("Received Link Anti-Entropy advertisement from peer: {}", message.sender());
+            LinkAntiEntropyAdvertisement advertisement = SERIALIZER.decode(message.payload());
+            handleAntiEntropyAdvertisement(advertisement);
+        }
+    }
+}

+package org.onlab.onos.store.link.impl;
+
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
+
+/**
+ * MessageSubjects used by GossipLinkStore peer-peer communication.
+ */
+public final class GossipLinkStoreMessageSubjects {
+
+    private GossipLinkStoreMessageSubjects() {}
+
+    public static final MessageSubject LINK_UPDATE =
+            new MessageSubject("peer-link-update");
+    public static final MessageSubject LINK_REMOVED =
+            new MessageSubject("peer-link-removed");
+    public static final MessageSubject LINK_ANTI_ENTROPY_ADVERTISEMENT =
+            new MessageSubject("link-enti-entropy-advertisement");
+}

+package org.onlab.onos.store.link.impl;
+
+import com.google.common.base.MoreObjects;
+
+import org.onlab.onos.net.link.LinkDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+/**
+ * Information published by GossipDeviceStore to notify peers of a device
+ * change event.
+ */
+public class InternalLinkEvent {
+
+    private final ProviderId providerId;
+    private final Timestamped<LinkDescription> linkDescription;
+
+    protected InternalLinkEvent(
+            ProviderId providerId,
+            Timestamped<LinkDescription> linkDescription) {
+        this.providerId = providerId;
+        this.linkDescription = linkDescription;
+    }
+
+    public ProviderId providerId() {
+        return providerId;
+    }
+
+    public Timestamped<LinkDescription> linkDescription() {
+        return linkDescription;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("linkDescription", linkDescription)
+                .toString();
+    }
+
+    // for serializer
+    protected InternalLinkEvent() {
+        this.providerId = null;
+        this.linkDescription = null;
+    }
+}

+package org.onlab.onos.store.link.impl;
+
+import org.onlab.onos.net.LinkKey;
+import org.onlab.onos.store.Timestamp;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Information published by GossipLinkStore to notify peers of a link
+ * being removed.
+ */
+public class InternalLinkRemovedEvent {
+
+    private final LinkKey linkKey;
+    private final Timestamp timestamp;
+
+    /**
+     * Creates a InternalLinkRemovedEvent.
+     * @param linkKey identifier of the removed link.
+     * @param timestamp timestamp of when the link was removed.
+     */
+    public InternalLinkRemovedEvent(LinkKey linkKey, Timestamp timestamp) {
+        this.linkKey = linkKey;
+        this.timestamp = timestamp;
+    }
+
+    public LinkKey linkKey() {
+        return linkKey;
+    }
+
+    public Timestamp timestamp() {
+        return timestamp;
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("linkKey", linkKey)
+                .add("timestamp", timestamp)
+                .toString();
+    }
+
+    // for serializer
+    @SuppressWarnings("unused")
+    private InternalLinkRemovedEvent() {
+        linkKey = null;
+        timestamp = null;
+    }
+}
\ No newline at end of file

+package org.onlab.onos.store.link.impl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Map;
+
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.net.LinkKey;
+import org.onlab.onos.store.Timestamp;
+
+/**
+ * Link AE Advertisement message.
+ */
+public class LinkAntiEntropyAdvertisement {
+
+    private final NodeId sender;
+    private final Map<LinkFragmentId, Timestamp> linkTimestamps;
+    private final Map<LinkKey, Timestamp> linkTombstones;
+
+
+    public LinkAntiEntropyAdvertisement(NodeId sender,
+                Map<LinkFragmentId, Timestamp> linkTimestamps,
+                Map<LinkKey, Timestamp> linkTombstones) {
+        this.sender = checkNotNull(sender);
+        this.linkTimestamps = checkNotNull(linkTimestamps);
+        this.linkTombstones = checkNotNull(linkTombstones);
+    }
+
+    public NodeId sender() {
+        return sender;
+    }
+
+    public Map<LinkFragmentId, Timestamp> linkTimestamps() {
+        return linkTimestamps;
+    }
+
+    public Map<LinkKey, Timestamp> linkTombstones() {
+        return linkTombstones;
+    }
+
+    // For serializer
+    @SuppressWarnings("unused")
+    private LinkAntiEntropyAdvertisement() {
+        this.sender = null;
+        this.linkTimestamps = null;
+        this.linkTombstones = null;
+    }
+}

+package org.onlab.onos.store.link.impl;
+
+import java.util.Objects;
+
+import org.onlab.onos.net.LinkKey;
+import org.onlab.onos.net.provider.ProviderId;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Identifier for LinkDescription from a Provider.
+ */
+public final class LinkFragmentId {
+    public final ProviderId providerId;
+    public final LinkKey linkKey;
+
+    public LinkFragmentId(LinkKey linkKey, ProviderId providerId) {
+        this.providerId = providerId;
+        this.linkKey = linkKey;
+    }
+
+    public LinkKey linkKey() {
+        return linkKey;
+    }
+
+    public ProviderId providerId() {
+        return providerId;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(providerId, linkKey);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof LinkFragmentId)) {
+            return false;
+        }
+        LinkFragmentId that = (LinkFragmentId) obj;
+        return Objects.equals(this.linkKey, that.linkKey) &&
+               Objects.equals(this.providerId, that.providerId);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("providerId", providerId)
+                .add("linkKey", linkKey)
+                .toString();
+    }
+
+    // for serializer
+    @SuppressWarnings("unused")
+    private LinkFragmentId() {
+        this.providerId = null;
+        this.linkKey = null;
+    }
+}

-package org.onlab.onos.store.link.impl;
-
-import static org.onlab.onos.net.Link.Type.DIRECT;
-import static org.onlab.onos.net.Link.Type.INDIRECT;
-import static org.onlab.onos.net.link.LinkEvent.Type.LINK_ADDED;
-import static org.onlab.onos.net.link.LinkEvent.Type.LINK_REMOVED;
-import static org.onlab.onos.net.link.LinkEvent.Type.LINK_UPDATED;
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import org.apache.felix.scr.annotations.Activate;
-import org.apache.felix.scr.annotations.Component;
-import org.apache.felix.scr.annotations.Deactivate;
-import org.apache.felix.scr.annotations.Reference;
-import org.apache.felix.scr.annotations.ReferenceCardinality;
-import org.apache.felix.scr.annotations.Service;
-import org.onlab.onos.net.ConnectPoint;
-import org.onlab.onos.net.DefaultLink;
-import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.net.Link;
-import org.onlab.onos.net.LinkKey;
-import org.onlab.onos.net.link.LinkDescription;
-import org.onlab.onos.net.link.LinkEvent;
-import org.onlab.onos.net.link.LinkStore;
-import org.onlab.onos.net.link.LinkStoreDelegate;
-import org.onlab.onos.net.provider.ProviderId;
-import org.onlab.onos.store.AbstractStore;
-import org.onlab.onos.store.ClockService;
-import org.onlab.onos.store.Timestamp;
-import org.onlab.onos.store.device.impl.VersionedValue;
-import org.slf4j.Logger;
-
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.ImmutableSet.Builder;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-//TODO: Add support for multiple provider and annotations
-/**
- * Manages inventory of infrastructure links using a protocol that takes into consideration
- * the order in which events occur.
- */
-// FIXME: This does not yet implement the full protocol.
-// The full protocol requires the sender of LLDP message to include the
-// version information of src device/port and the receiver to
-// take that into account when figuring out if a more recent src
-// device/port down event renders the link discovery obsolete.
-@Component(immediate = true)
-@Service
-public class OnosDistributedLinkStore
-    extends AbstractStore<LinkEvent, LinkStoreDelegate>
-    implements LinkStore {
-
-    private final Logger log = getLogger(getClass());
-
-    // Link inventory
-    private ConcurrentMap<LinkKey, VersionedValue<Link>> links;
-
-    public static final String LINK_NOT_FOUND = "Link between %s and %s not found";
-
-    // TODO synchronize?
-    // Egress and ingress link sets
-    private final Multimap<DeviceId, VersionedValue<Link>> srcLinks = HashMultimap.create();
-    private final Multimap<DeviceId, VersionedValue<Link>> dstLinks = HashMultimap.create();
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected ClockService clockService;
-
-    @Activate
-    public void activate() {
-
-        links = new ConcurrentHashMap<>();
-
-        log.info("Started");
-    }
-
-    @Deactivate
-    public void deactivate() {
-        log.info("Stopped");
-    }
-
-    @Override
-    public int getLinkCount() {
-        return links.size();
-    }
-
-    @Override
-    public Iterable<Link> getLinks() {
-        Builder<Link> builder = ImmutableSet.builder();
-        synchronized (this) {
-            for (VersionedValue<Link> link : links.values()) {
-                builder.add(link.entity());
-            }
-            return builder.build();
-        }
-    }
-
-    @Override
-    public Set<Link> getDeviceEgressLinks(DeviceId deviceId) {
-        Set<VersionedValue<Link>> egressLinks = ImmutableSet.copyOf(srcLinks.get(deviceId));
-        Set<Link> rawEgressLinks = new HashSet<>();
-        for (VersionedValue<Link> link : egressLinks) {
-            rawEgressLinks.add(link.entity());
-        }
-        return rawEgressLinks;
-    }
-
-    @Override
-    public Set<Link> getDeviceIngressLinks(DeviceId deviceId) {
-        Set<VersionedValue<Link>> ingressLinks = ImmutableSet.copyOf(dstLinks.get(deviceId));
-        Set<Link> rawIngressLinks = new HashSet<>();
-        for (VersionedValue<Link> link : ingressLinks) {
-            rawIngressLinks.add(link.entity());
-        }
-        return rawIngressLinks;
-    }
-
-    @Override
-    public Link getLink(ConnectPoint src, ConnectPoint dst) {
-        VersionedValue<Link> link = links.get(new LinkKey(src, dst));
-        checkArgument(link != null, "LINK_NOT_FOUND", src, dst);
-        return link.entity();
-    }
-
-    @Override
-    public Set<Link> getEgressLinks(ConnectPoint src) {
-        Set<Link> egressLinks = new HashSet<>();
-        for (VersionedValue<Link> link : srcLinks.get(src.deviceId())) {
-            if (link.entity().src().equals(src)) {
-                egressLinks.add(link.entity());
-            }
-        }
-        return egressLinks;
-    }
-
-    @Override
-    public Set<Link> getIngressLinks(ConnectPoint dst) {
-        Set<Link> ingressLinks = new HashSet<>();
-        for (VersionedValue<Link> link : dstLinks.get(dst.deviceId())) {
-            if (link.entity().dst().equals(dst)) {
-                ingressLinks.add(link.entity());
-            }
-        }
-        return ingressLinks;
-    }
-
-    @Override
-    public LinkEvent createOrUpdateLink(ProviderId providerId,
-                                        LinkDescription linkDescription) {
-
-        final DeviceId destinationDeviceId = linkDescription.dst().deviceId();
-        final Timestamp newTimestamp = clockService.getTimestamp(destinationDeviceId);
-
-        LinkKey key = new LinkKey(linkDescription.src(), linkDescription.dst());
-        VersionedValue<Link> link = links.get(key);
-        if (link == null) {
-            return createLink(providerId, key, linkDescription, newTimestamp);
-        }
-
-        checkState(newTimestamp.compareTo(link.timestamp()) > 0,
-                "Existing Link has a timestamp in the future!");
-
-        return updateLink(providerId, link, key, linkDescription, newTimestamp);
-    }
-
-    // Creates and stores the link and returns the appropriate event.
-    private LinkEvent createLink(ProviderId providerId, LinkKey key,
-            LinkDescription linkDescription, Timestamp timestamp) {
-        VersionedValue<Link> link = new VersionedValue<Link>(new DefaultLink(providerId, key.src(), key.dst(),
-                linkDescription.type()), true, timestamp);
-        synchronized (this) {
-            links.put(key, link);
-            addNewLink(link, timestamp);
-        }
-        // FIXME: notify peers.
-        return new LinkEvent(LINK_ADDED, link.entity());
-    }
-
-    // update Egress and ingress link sets
-    private void addNewLink(VersionedValue<Link> link, Timestamp timestamp) {
-        Link rawLink = link.entity();
-        synchronized (this) {
-            srcLinks.put(rawLink.src().deviceId(), link);
-            dstLinks.put(rawLink.dst().deviceId(), link);
-        }
-    }
-
-    // Updates, if necessary the specified link and returns the appropriate event.
-    private LinkEvent updateLink(ProviderId providerId, VersionedValue<Link> existingLink,
-                                 LinkKey key, LinkDescription linkDescription, Timestamp timestamp) {
-        // FIXME confirm Link update condition is OK
-        if (existingLink.entity().type() == INDIRECT && linkDescription.type() == DIRECT) {
-            synchronized (this) {
-
-                VersionedValue<Link> updatedLink = new VersionedValue<Link>(
-                        new DefaultLink(providerId, existingLink.entity().src(), existingLink.entity().dst(),
-                                        linkDescription.type()), true, timestamp);
-                links.replace(key, existingLink, updatedLink);
-
-                replaceLink(existingLink, updatedLink);
-                // FIXME: notify peers.
-                return new LinkEvent(LINK_UPDATED, updatedLink.entity());
-            }
-        }
-        return null;
-    }
-
-    // update Egress and ingress link sets
-    private void replaceLink(VersionedValue<Link> current, VersionedValue<Link> updated) {
-        synchronized (this) {
-            srcLinks.remove(current.entity().src().deviceId(), current);
-            dstLinks.remove(current.entity().dst().deviceId(), current);
-
-            srcLinks.put(current.entity().src().deviceId(), updated);
-            dstLinks.put(current.entity().dst().deviceId(), updated);
-        }
-    }
-
-    @Override
-    public LinkEvent removeLink(ConnectPoint src, ConnectPoint dst) {
-        synchronized (this) {
-            LinkKey key = new LinkKey(src, dst);
-            VersionedValue<Link> link = links.remove(key);
-            if (link != null) {
-                removeLink(link);
-                // notify peers
-                return new LinkEvent(LINK_REMOVED, link.entity());
-            }
-            return null;
-        }
-    }
-
-    // update Egress and ingress link sets
-    private void removeLink(VersionedValue<Link> link) {
-        synchronized (this) {
-            srcLinks.remove(link.entity().src().deviceId(), link);
-            dstLinks.remove(link.entity().dst().deviceId(), link);
-        }
-    }
-}

@@ -35,4 +35,4 @@ public final class ClusterMessageSerializer extends Serializer<ClusterMessage> {
         byte[] payload = input.readBytes(payloadSize);
         byte[] payload = input.readBytes(payloadSize);
         return new ClusterMessage(sender, subject, payload);
         return new ClusterMessage(sender, subject, payload);
     }
     }
-}
\ No newline at end of file
+}

+package org.onlab.onos.store.serializers;
+
+import org.onlab.onos.store.common.impl.MastershipBasedTimestamp;
+import org.onlab.onos.store.common.impl.Timestamped;
+import org.onlab.util.KryoPool;
+
+public final class DistributedStoreSerializers {
+
+    /**
+     * KryoPool which can serialize ON.lab misc classes.
+     */
+    public static final KryoPool COMMON = KryoPool.newBuilder()
+            .register(KryoPoolUtil.API)
+            .register(Timestamped.class)
+            .register(MastershipBasedTimestamp.class, new MastershipBasedTimestampSerializer())
+            .build();
+
+    // avoid instantiation
+    private DistributedStoreSerializers() {}
+}

 package org.onlab.onos.store.device.impl;
 package org.onlab.onos.store.device.impl;
 
 
+import static org.easymock.EasyMock.*;
 import static org.junit.Assert.*;
 import static org.junit.Assert.*;
 import static org.onlab.onos.net.Device.Type.SWITCH;
 import static org.onlab.onos.net.Device.Type.SWITCH;
 import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.onlab.onos.net.device.DeviceEvent.Type.*;
 import static org.onlab.onos.net.device.DeviceEvent.Type.*;
-
+import static org.onlab.onos.cluster.ControllerNode.State.*;
+import static org.onlab.onos.net.DefaultAnnotations.union;
+import static java.util.Arrays.asList;
 import java.io.IOException;
 import java.io.IOException;
 import java.util.Arrays;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashMap;
 import java.util.List;
 import java.util.List;
 import java.util.Map;
 import java.util.Map;
@@ -14,6 +18,7 @@ import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.TimeUnit;
 
 
+import org.easymock.Capture;
 import org.junit.After;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.Before;
@@ -90,14 +95,25 @@ public class GossipDeviceStoreTest {
             .set("B4", "b4")
             .set("B4", "b4")
             .build();
             .build();
 
 
-    private static final NodeId MYSELF = new NodeId("myself");
+    // local node
+    private static final NodeId NID1 = new NodeId("local");
+    private static final ControllerNode ONOS1 =
+            new DefaultControllerNode(NID1, IpPrefix.valueOf("127.0.0.1"));
+
+    // remote node
+    private static final NodeId NID2 = new NodeId("remote");
+    private static final ControllerNode ONOS2 =
+            new DefaultControllerNode(NID2, IpPrefix.valueOf("127.0.0.2"));
+    private static final List<SparseAnnotations> NO_ANNOTATION = Collections.<SparseAnnotations>emptyList();
 
 
+
+    private TestGossipDeviceStore testGossipDeviceStore;
     private GossipDeviceStore gossipDeviceStore;
     private GossipDeviceStore gossipDeviceStore;
     private DeviceStore deviceStore;
     private DeviceStore deviceStore;
 
 
     private DeviceClockManager deviceClockManager;
     private DeviceClockManager deviceClockManager;
     private ClockService clockService;
     private ClockService clockService;
-
+    private ClusterCommunicationService clusterCommunicator;
     @BeforeClass
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
     public static void setUpBeforeClass() throws Exception {
     }
     }
@@ -113,15 +129,22 @@ public class GossipDeviceStoreTest {
         deviceClockManager.activate();
         deviceClockManager.activate();
         clockService = deviceClockManager;
         clockService = deviceClockManager;
 
 
-        deviceClockManager.setMastershipTerm(DID1, MastershipTerm.of(MYSELF, 1));
-        deviceClockManager.setMastershipTerm(DID2, MastershipTerm.of(MYSELF, 2));
+        deviceClockManager.setMastershipTerm(DID1, MastershipTerm.of(NID1, 1));
+        deviceClockManager.setMastershipTerm(DID2, MastershipTerm.of(NID1, 2));
 
 
-        ClusterCommunicationService clusterCommunicator = new TestClusterCommunicationService();
+        clusterCommunicator = createNiceMock(ClusterCommunicationService.class);
+        clusterCommunicator.addSubscriber(anyObject(MessageSubject.class),
+                                    anyObject(ClusterMessageHandler.class));
+        expectLastCall().anyTimes();
+        replay(clusterCommunicator);
         ClusterService clusterService = new TestClusterService();
         ClusterService clusterService = new TestClusterService();
 
 
-        gossipDeviceStore = new TestGossipDeviceStore(clockService, clusterService, clusterCommunicator);
+        testGossipDeviceStore = new TestGossipDeviceStore(clockService, clusterService, clusterCommunicator);
+        gossipDeviceStore = testGossipDeviceStore;
         gossipDeviceStore.activate();
         gossipDeviceStore.activate();
         deviceStore = gossipDeviceStore;
         deviceStore = gossipDeviceStore;
+        verify(clusterCommunicator);
+        reset(clusterCommunicator);
     }
     }
 
 
     @After
     @After
@@ -135,7 +158,16 @@ public class GossipDeviceStoreTest {
         DeviceDescription description =
         DeviceDescription description =
                 new DefaultDeviceDescription(deviceId.uri(), SWITCH, MFR,
                 new DefaultDeviceDescription(deviceId.uri(), SWITCH, MFR,
                         HW, swVersion, SN, annotations);
                         HW, swVersion, SN, annotations);
+        reset(clusterCommunicator);
+        try {
+            expect(clusterCommunicator.broadcast(anyObject(ClusterMessage.class)))
+                .andReturn(true).anyTimes();
+        } catch (IOException e) {
+            fail("Should never reach here");
+        }
+        replay(clusterCommunicator);
         deviceStore.createOrUpdateDevice(PID, deviceId, description);
         deviceStore.createOrUpdateDevice(PID, deviceId, description);
+        verify(clusterCommunicator);
     }
     }
 
 
     private void putDeviceAncillary(DeviceId deviceId, String swVersion,
     private void putDeviceAncillary(DeviceId deviceId, String swVersion,
@@ -163,9 +195,9 @@ public class GossipDeviceStoreTest {
      * @param annotations
      * @param annotations
      */
      */
     private static void assertAnnotationsEquals(Annotations actual, SparseAnnotations... annotations) {
     private static void assertAnnotationsEquals(Annotations actual, SparseAnnotations... annotations) {
-        DefaultAnnotations expected = DefaultAnnotations.builder().build();
+        SparseAnnotations expected = DefaultAnnotations.builder().build();
         for (SparseAnnotations a : annotations) {
         for (SparseAnnotations a : annotations) {
-            expected = DefaultAnnotations.merge(expected, a);
+            expected = DefaultAnnotations.union(expected, a);
         }
         }
         assertEquals(expected.keys(), actual.keys());
         assertEquals(expected.keys(), actual.keys());
         for (String key : expected.keys()) {
         for (String key : expected.keys()) {
@@ -173,6 +205,36 @@ public class GossipDeviceStoreTest {
         }
         }
     }
     }
 
 
+    private static void assertDeviceDescriptionEquals(DeviceDescription expected,
+                                                DeviceDescription actual) {
+        if (expected == actual) {
+            return;
+        }
+        assertEquals(expected.deviceURI(), actual.deviceURI());
+        assertEquals(expected.hwVersion(), actual.hwVersion());
+        assertEquals(expected.manufacturer(), actual.manufacturer());
+        assertEquals(expected.serialNumber(), actual.serialNumber());
+        assertEquals(expected.swVersion(), actual.swVersion());
+
+        assertAnnotationsEquals(actual.annotations(), expected.annotations());
+    }
+
+    private static void assertDeviceDescriptionEquals(DeviceDescription expected,
+            List<SparseAnnotations> expectedAnnotations,
+            DeviceDescription actual) {
+        if (expected == actual) {
+            return;
+        }
+        assertEquals(expected.deviceURI(), actual.deviceURI());
+        assertEquals(expected.hwVersion(), actual.hwVersion());
+        assertEquals(expected.manufacturer(), actual.manufacturer());
+        assertEquals(expected.serialNumber(), actual.serialNumber());
+        assertEquals(expected.swVersion(), actual.swVersion());
+
+        assertAnnotationsEquals(actual.annotations(),
+                expectedAnnotations.toArray(new SparseAnnotations[0]));
+    }
+
     @Test
     @Test
     public final void testGetDeviceCount() {
     public final void testGetDeviceCount() {
         assertEquals("initialy empty", 0, deviceStore.getDeviceCount());
         assertEquals("initialy empty", 0, deviceStore.getDeviceCount());
@@ -215,56 +277,123 @@ public class GossipDeviceStoreTest {
         assertNull("DID2 shouldn't be there", deviceStore.getDevice(DID2));
         assertNull("DID2 shouldn't be there", deviceStore.getDevice(DID2));
     }
     }
 
 
+    private void assertInternalDeviceEvent(NodeId sender,
+                                           DeviceId deviceId,
+                                           ProviderId providerId,
+                                           DeviceDescription expectedDesc,
+                                           Capture<ClusterMessage> actualMsg) {
+        assertTrue(actualMsg.hasCaptured());
+        assertEquals(sender, actualMsg.getValue().sender());
+        assertEquals(GossipDeviceStoreMessageSubjects.DEVICE_UPDATE,
+                     actualMsg.getValue().subject());
+        InternalDeviceEvent addEvent
+            = testGossipDeviceStore.deserialize(actualMsg.getValue().payload());
+        assertEquals(deviceId, addEvent.deviceId());
+        assertEquals(providerId, addEvent.providerId());
+        assertDeviceDescriptionEquals(expectedDesc, addEvent.deviceDescription().value());
+    }
+
+    private void assertInternalDeviceEvent(NodeId sender,
+                                           DeviceId deviceId,
+                                           ProviderId providerId,
+                                           DeviceDescription expectedDesc,
+                                           List<SparseAnnotations> expectedAnnotations,
+                                           Capture<ClusterMessage> actualMsg) {
+        assertTrue(actualMsg.hasCaptured());
+        assertEquals(sender, actualMsg.getValue().sender());
+        assertEquals(GossipDeviceStoreMessageSubjects.DEVICE_UPDATE,
+                actualMsg.getValue().subject());
+        InternalDeviceEvent addEvent
+            = testGossipDeviceStore.deserialize(actualMsg.getValue().payload());
+        assertEquals(deviceId, addEvent.deviceId());
+        assertEquals(providerId, addEvent.providerId());
+        assertDeviceDescriptionEquals(expectedDesc, expectedAnnotations, addEvent.deviceDescription().value());
+    }
+
     @Test
     @Test
-    public final void testCreateOrUpdateDevice() {
+    public final void testCreateOrUpdateDevice() throws IOException {
         DeviceDescription description =
         DeviceDescription description =
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                         HW, SW1, SN);
                         HW, SW1, SN);
+        Capture<ClusterMessage> bcast = new Capture<>();
+
+        resetCommunicatorExpectingSingleBroadcast(bcast);
         DeviceEvent event = deviceStore.createOrUpdateDevice(PID, DID1, description);
         DeviceEvent event = deviceStore.createOrUpdateDevice(PID, DID1, description);
         assertEquals(DEVICE_ADDED, event.type());
         assertEquals(DEVICE_ADDED, event.type());
         assertDevice(DID1, SW1, event.subject());
         assertDevice(DID1, SW1, event.subject());
+        verify(clusterCommunicator);
+        assertInternalDeviceEvent(NID1, DID1, PID, description, bcast);
+
 
 
         DeviceDescription description2 =
         DeviceDescription description2 =
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                         HW, SW2, SN);
                         HW, SW2, SN);
+        resetCommunicatorExpectingSingleBroadcast(bcast);
         DeviceEvent event2 = deviceStore.createOrUpdateDevice(PID, DID1, description2);
         DeviceEvent event2 = deviceStore.createOrUpdateDevice(PID, DID1, description2);
         assertEquals(DEVICE_UPDATED, event2.type());
         assertEquals(DEVICE_UPDATED, event2.type());
         assertDevice(DID1, SW2, event2.subject());
         assertDevice(DID1, SW2, event2.subject());
 
 
+        verify(clusterCommunicator);
+        assertInternalDeviceEvent(NID1, DID1, PID, description2, bcast);
+        reset(clusterCommunicator);
+
         assertNull("No change expected", deviceStore.createOrUpdateDevice(PID, DID1, description2));
         assertNull("No change expected", deviceStore.createOrUpdateDevice(PID, DID1, description2));
     }
     }
 
 
     @Test
     @Test
-    public final void testCreateOrUpdateDeviceAncillary() {
+    public final void testCreateOrUpdateDeviceAncillary() throws IOException {
+        // add
         DeviceDescription description =
         DeviceDescription description =
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                         HW, SW1, SN, A2);
                         HW, SW1, SN, A2);
+        Capture<ClusterMessage> bcast = new Capture<>();
+
+        resetCommunicatorExpectingSingleBroadcast(bcast);
         DeviceEvent event = deviceStore.createOrUpdateDevice(PIDA, DID1, description);
         DeviceEvent event = deviceStore.createOrUpdateDevice(PIDA, DID1, description);
         assertEquals(DEVICE_ADDED, event.type());
         assertEquals(DEVICE_ADDED, event.type());
         assertDevice(DID1, SW1, event.subject());
         assertDevice(DID1, SW1, event.subject());
         assertEquals(PIDA, event.subject().providerId());
         assertEquals(PIDA, event.subject().providerId());
         assertAnnotationsEquals(event.subject().annotations(), A2);
         assertAnnotationsEquals(event.subject().annotations(), A2);
         assertFalse("Ancillary will not bring device up", deviceStore.isAvailable(DID1));
         assertFalse("Ancillary will not bring device up", deviceStore.isAvailable(DID1));
+        verify(clusterCommunicator);
+        assertInternalDeviceEvent(NID1, DID1, PIDA, description, bcast);
 
 
+        // update from primary
         DeviceDescription description2 =
         DeviceDescription description2 =
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                         HW, SW2, SN, A1);
                         HW, SW2, SN, A1);
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+
         DeviceEvent event2 = deviceStore.createOrUpdateDevice(PID, DID1, description2);
         DeviceEvent event2 = deviceStore.createOrUpdateDevice(PID, DID1, description2);
         assertEquals(DEVICE_UPDATED, event2.type());
         assertEquals(DEVICE_UPDATED, event2.type());
         assertDevice(DID1, SW2, event2.subject());
         assertDevice(DID1, SW2, event2.subject());
         assertEquals(PID, event2.subject().providerId());
         assertEquals(PID, event2.subject().providerId());
         assertAnnotationsEquals(event2.subject().annotations(), A1, A2);
         assertAnnotationsEquals(event2.subject().annotations(), A1, A2);
         assertTrue(deviceStore.isAvailable(DID1));
         assertTrue(deviceStore.isAvailable(DID1));
+        verify(clusterCommunicator);
+        assertInternalDeviceEvent(NID1, DID1, PID, description2, bcast);
 
 
+        // no-op update from primary
+        resetCommunicatorExpectingNoBroadcast(bcast);
         assertNull("No change expected", deviceStore.createOrUpdateDevice(PID, DID1, description2));
         assertNull("No change expected", deviceStore.createOrUpdateDevice(PID, DID1, description2));
 
 
+        verify(clusterCommunicator);
+        assertFalse("no broadcast expected", bcast.hasCaptured());
+
         // For now, Ancillary is ignored once primary appears
         // For now, Ancillary is ignored once primary appears
+        resetCommunicatorExpectingNoBroadcast(bcast);
+
         assertNull("No change expected", deviceStore.createOrUpdateDevice(PIDA, DID1, description));
         assertNull("No change expected", deviceStore.createOrUpdateDevice(PIDA, DID1, description));
 
 
+        verify(clusterCommunicator);
+        assertFalse("no broadcast expected", bcast.hasCaptured());
+
         // But, Ancillary annotations will be in effect
         // But, Ancillary annotations will be in effect
         DeviceDescription description3 =
         DeviceDescription description3 =
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                 new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
                         HW, SW1, SN, A2_2);
                         HW, SW1, SN, A2_2);
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+
         DeviceEvent event3 = deviceStore.createOrUpdateDevice(PIDA, DID1, description3);
         DeviceEvent event3 = deviceStore.createOrUpdateDevice(PIDA, DID1, description3);
         assertEquals(DEVICE_UPDATED, event3.type());
         assertEquals(DEVICE_UPDATED, event3.type());
         // basic information will be the one from Primary
         // basic information will be the one from Primary
@@ -273,6 +402,11 @@ public class GossipDeviceStoreTest {
         // but annotation from Ancillary will be merged
         // but annotation from Ancillary will be merged
         assertAnnotationsEquals(event3.subject().annotations(), A1, A2, A2_2);
         assertAnnotationsEquals(event3.subject().annotations(), A1, A2, A2_2);
         assertTrue(deviceStore.isAvailable(DID1));
         assertTrue(deviceStore.isAvailable(DID1));
+        verify(clusterCommunicator);
+        // note: only annotation from PIDA is sent over the wire
+        assertInternalDeviceEvent(NID1, DID1, PIDA, description3,
+                                  asList(union(A2, A2_2)), bcast);
+
     }
     }
 
 
 
 
@@ -282,14 +416,24 @@ public class GossipDeviceStoreTest {
         putDevice(DID1, SW1);
         putDevice(DID1, SW1);
         assertTrue(deviceStore.isAvailable(DID1));
         assertTrue(deviceStore.isAvailable(DID1));
 
 
+        Capture<ClusterMessage> bcast = new Capture<>();
+
+        resetCommunicatorExpectingSingleBroadcast(bcast);
         DeviceEvent event = deviceStore.markOffline(DID1);
         DeviceEvent event = deviceStore.markOffline(DID1);
         assertEquals(DEVICE_AVAILABILITY_CHANGED, event.type());
         assertEquals(DEVICE_AVAILABILITY_CHANGED, event.type());
         assertDevice(DID1, SW1, event.subject());
         assertDevice(DID1, SW1, event.subject());
         assertFalse(deviceStore.isAvailable(DID1));
         assertFalse(deviceStore.isAvailable(DID1));
+        verify(clusterCommunicator);
+        // TODO: verify broadcast message
+        assertTrue(bcast.hasCaptured());
+
 
 
+        resetCommunicatorExpectingNoBroadcast(bcast);
         DeviceEvent event2 = deviceStore.markOffline(DID1);
         DeviceEvent event2 = deviceStore.markOffline(DID1);
         assertNull("No change, no event", event2);
         assertNull("No change, no event", event2);
-}
+        verify(clusterCommunicator);
+        assertFalse(bcast.hasCaptured());
+    }
 
 
     @Test
     @Test
     public final void testUpdatePorts() {
     public final void testUpdatePorts() {
@@ -298,8 +442,13 @@ public class GossipDeviceStoreTest {
                 new DefaultPortDescription(P1, true),
                 new DefaultPortDescription(P1, true),
                 new DefaultPortDescription(P2, true)
                 new DefaultPortDescription(P2, true)
                 );
                 );
+        Capture<ClusterMessage> bcast = new Capture<>();
 
 
+        resetCommunicatorExpectingSingleBroadcast(bcast);
         List<DeviceEvent> events = deviceStore.updatePorts(PID, DID1, pds);
         List<DeviceEvent> events = deviceStore.updatePorts(PID, DID1, pds);
+        verify(clusterCommunicator);
+        // TODO: verify broadcast message
+        assertTrue(bcast.hasCaptured());
 
 
         Set<PortNumber> expectedPorts = Sets.newHashSet(P1, P2);
         Set<PortNumber> expectedPorts = Sets.newHashSet(P1, P2);
         for (DeviceEvent event : events) {
         for (DeviceEvent event : events) {
@@ -318,7 +467,12 @@ public class GossipDeviceStoreTest {
                 new DefaultPortDescription(P3, true)
                 new DefaultPortDescription(P3, true)
                 );
                 );
 
 
+        resetCommunicatorExpectingSingleBroadcast(bcast);
         events = deviceStore.updatePorts(PID, DID1, pds2);
         events = deviceStore.updatePorts(PID, DID1, pds2);
+        verify(clusterCommunicator);
+        // TODO: verify broadcast message
+        assertTrue(bcast.hasCaptured());
+
         assertFalse("event should be triggered", events.isEmpty());
         assertFalse("event should be triggered", events.isEmpty());
         for (DeviceEvent event : events) {
         for (DeviceEvent event : events) {
             PortNumber num = event.port().number();
             PortNumber num = event.port().number();
@@ -341,7 +495,12 @@ public class GossipDeviceStoreTest {
                 new DefaultPortDescription(P1, false),
                 new DefaultPortDescription(P1, false),
                 new DefaultPortDescription(P2, true)
                 new DefaultPortDescription(P2, true)
                 );
                 );
+        resetCommunicatorExpectingSingleBroadcast(bcast);
         events = deviceStore.updatePorts(PID, DID1, pds3);
         events = deviceStore.updatePorts(PID, DID1, pds3);
+        verify(clusterCommunicator);
+        // TODO: verify broadcast message
+        assertTrue(bcast.hasCaptured());
+
         assertFalse("event should be triggered", events.isEmpty());
         assertFalse("event should be triggered", events.isEmpty());
         for (DeviceEvent event : events) {
         for (DeviceEvent event : events) {
             PortNumber num = event.port().number();
             PortNumber num = event.port().number();
@@ -357,7 +516,6 @@ public class GossipDeviceStoreTest {
                 fail("Unknown port number encountered: " + num);
                 fail("Unknown port number encountered: " + num);
             }
             }
         }
         }
-
     }
     }
 
 
     @Test
     @Test
@@ -368,16 +526,22 @@ public class GossipDeviceStoreTest {
                 );
                 );
         deviceStore.updatePorts(PID, DID1, pds);
         deviceStore.updatePorts(PID, DID1, pds);
 
 
-        DeviceEvent event = deviceStore.updatePortStatus(PID, DID1,
-                new DefaultPortDescription(P1, false));
+        Capture<ClusterMessage> bcast = new Capture<>();
+
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        final DefaultPortDescription desc = new DefaultPortDescription(P1, false);
+        DeviceEvent event = deviceStore.updatePortStatus(PID, DID1, desc);
         assertEquals(PORT_UPDATED, event.type());
         assertEquals(PORT_UPDATED, event.type());
         assertDevice(DID1, SW1, event.subject());
         assertDevice(DID1, SW1, event.subject());
         assertEquals(P1, event.port().number());
         assertEquals(P1, event.port().number());
         assertFalse("Port is disabled", event.port().isEnabled());
         assertFalse("Port is disabled", event.port().isEnabled());
-
+        verify(clusterCommunicator);
+        assertInternalPortStatusEvent(NID1, DID1, PID, desc, NO_ANNOTATION, bcast);
+        assertTrue(bcast.hasCaptured());
     }
     }
+
     @Test
     @Test
-    public final void testUpdatePortStatusAncillary() {
+    public final void testUpdatePortStatusAncillary() throws IOException {
         putDeviceAncillary(DID1, SW1);
         putDeviceAncillary(DID1, SW1);
         putDevice(DID1, SW1);
         putDevice(DID1, SW1);
         List<PortDescription> pds = Arrays.<PortDescription>asList(
         List<PortDescription> pds = Arrays.<PortDescription>asList(
@@ -385,36 +549,106 @@ public class GossipDeviceStoreTest {
                 );
                 );
         deviceStore.updatePorts(PID, DID1, pds);
         deviceStore.updatePorts(PID, DID1, pds);
 
 
-        DeviceEvent event = deviceStore.updatePortStatus(PID, DID1,
-                new DefaultPortDescription(P1, false, A1_2));
+        Capture<ClusterMessage> bcast = new Capture<>();
+
+
+        // update port from primary
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        final DefaultPortDescription desc1 = new DefaultPortDescription(P1, false, A1_2);
+        DeviceEvent event = deviceStore.updatePortStatus(PID, DID1, desc1);
         assertEquals(PORT_UPDATED, event.type());
         assertEquals(PORT_UPDATED, event.type());
         assertDevice(DID1, SW1, event.subject());
         assertDevice(DID1, SW1, event.subject());
         assertEquals(P1, event.port().number());
         assertEquals(P1, event.port().number());
         assertAnnotationsEquals(event.port().annotations(), A1, A1_2);
         assertAnnotationsEquals(event.port().annotations(), A1, A1_2);
         assertFalse("Port is disabled", event.port().isEnabled());
         assertFalse("Port is disabled", event.port().isEnabled());
-
-        DeviceEvent event2 = deviceStore.updatePortStatus(PIDA, DID1,
-                new DefaultPortDescription(P1, true));
+        verify(clusterCommunicator);
+        assertInternalPortStatusEvent(NID1, DID1, PID, desc1, asList(A1, A1_2), bcast);
+        assertTrue(bcast.hasCaptured());
+
+        // update port from ancillary with no attributes
+        resetCommunicatorExpectingNoBroadcast(bcast);
+        final DefaultPortDescription desc2 = new DefaultPortDescription(P1, true);
+        DeviceEvent event2 = deviceStore.updatePortStatus(PIDA, DID1, desc2);
         assertNull("Ancillary is ignored if primary exists", event2);
         assertNull("Ancillary is ignored if primary exists", event2);
+        verify(clusterCommunicator);
+        assertFalse(bcast.hasCaptured());
 
 
         // but, Ancillary annotation update will be notified
         // but, Ancillary annotation update will be notified
-        DeviceEvent event3 = deviceStore.updatePortStatus(PIDA, DID1,
-                new DefaultPortDescription(P1, true, A2));
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        final DefaultPortDescription desc3 = new DefaultPortDescription(P1, true, A2);
+        DeviceEvent event3 = deviceStore.updatePortStatus(PIDA, DID1, desc3);
         assertEquals(PORT_UPDATED, event3.type());
         assertEquals(PORT_UPDATED, event3.type());
         assertDevice(DID1, SW1, event3.subject());
         assertDevice(DID1, SW1, event3.subject());
         assertEquals(P1, event3.port().number());
         assertEquals(P1, event3.port().number());
         assertAnnotationsEquals(event3.port().annotations(), A1, A1_2, A2);
         assertAnnotationsEquals(event3.port().annotations(), A1, A1_2, A2);
         assertFalse("Port is disabled", event3.port().isEnabled());
         assertFalse("Port is disabled", event3.port().isEnabled());
+        verify(clusterCommunicator);
+        assertInternalPortStatusEvent(NID1, DID1, PIDA, desc3, asList(A2), bcast);
+        assertTrue(bcast.hasCaptured());
 
 
         // port only reported from Ancillary will be notified as down
         // port only reported from Ancillary will be notified as down
-        DeviceEvent event4 = deviceStore.updatePortStatus(PIDA, DID1,
-                new DefaultPortDescription(P2, true));
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+        final DefaultPortDescription desc4 = new DefaultPortDescription(P2, true);
+        DeviceEvent event4 = deviceStore.updatePortStatus(PIDA, DID1, desc4);
         assertEquals(PORT_ADDED, event4.type());
         assertEquals(PORT_ADDED, event4.type());
         assertDevice(DID1, SW1, event4.subject());
         assertDevice(DID1, SW1, event4.subject());
         assertEquals(P2, event4.port().number());
         assertEquals(P2, event4.port().number());
         assertAnnotationsEquals(event4.port().annotations());
         assertAnnotationsEquals(event4.port().annotations());
         assertFalse("Port is disabled if not given from primary provider",
         assertFalse("Port is disabled if not given from primary provider",
                         event4.port().isEnabled());
                         event4.port().isEnabled());
+        verify(clusterCommunicator);
+        // TODO: verify broadcast message content
+        assertInternalPortStatusEvent(NID1, DID1, PIDA, desc4, NO_ANNOTATION, bcast);
+        assertTrue(bcast.hasCaptured());
+    }
+
+    private void assertInternalPortStatusEvent(NodeId sender, DeviceId did,
+            ProviderId pid, DefaultPortDescription expectedDesc,
+            List<SparseAnnotations> expectedAnnotations, Capture<ClusterMessage> actualMsg) {
+
+        assertTrue(actualMsg.hasCaptured());
+        assertEquals(sender, actualMsg.getValue().sender());
+        assertEquals(GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE,
+                actualMsg.getValue().subject());
+        InternalPortStatusEvent addEvent
+            = testGossipDeviceStore.deserialize(actualMsg.getValue().payload());
+        assertEquals(did, addEvent.deviceId());
+        assertEquals(pid, addEvent.providerId());
+        assertPortDescriptionEquals(expectedDesc, expectedAnnotations,
+                addEvent.portDescription().value());
+
+    }
+
+    private void assertPortDescriptionEquals(
+                                    PortDescription expectedDesc,
+                                    List<SparseAnnotations> expectedAnnotations,
+                                    PortDescription actual) {
+
+        assertEquals(expectedDesc.portNumber(), actual.portNumber());
+        assertEquals(expectedDesc.isEnabled(), actual.isEnabled());
+
+        assertAnnotationsEquals(actual.annotations(),
+                         expectedAnnotations.toArray(new SparseAnnotations[0]));
+    }
+
+    private void resetCommunicatorExpectingNoBroadcast(
+            Capture<ClusterMessage> bcast) {
+        bcast.reset();
+        reset(clusterCommunicator);
+        replay(clusterCommunicator);
+    }
+
+    private void resetCommunicatorExpectingSingleBroadcast(
+            Capture<ClusterMessage> bcast) {
+
+        bcast.reset();
+        reset(clusterCommunicator);
+        try {
+            expect(clusterCommunicator.broadcast(capture(bcast))).andReturn(true).once();
+        } catch (IOException e) {
+            fail("Should never reach here");
+        }
+        replay(clusterCommunicator);
     }
     }
 
 
     @Test
     @Test
@@ -476,12 +710,19 @@ public class GossipDeviceStoreTest {
         assertAnnotationsEquals(deviceStore.getDevice(DID1).annotations(), A1);
         assertAnnotationsEquals(deviceStore.getDevice(DID1).annotations(), A1);
         assertAnnotationsEquals(deviceStore.getPort(DID1, P1).annotations(), A2);
         assertAnnotationsEquals(deviceStore.getPort(DID1, P1).annotations(), A2);
 
 
+        Capture<ClusterMessage> bcast = new Capture<>();
+
+        resetCommunicatorExpectingSingleBroadcast(bcast);
+
         DeviceEvent event = deviceStore.removeDevice(DID1);
         DeviceEvent event = deviceStore.removeDevice(DID1);
         assertEquals(DEVICE_REMOVED, event.type());
         assertEquals(DEVICE_REMOVED, event.type());
         assertDevice(DID1, SW1, event.subject());
         assertDevice(DID1, SW1, event.subject());
 
 
         assertEquals(1, deviceStore.getDeviceCount());
         assertEquals(1, deviceStore.getDeviceCount());
         assertEquals(0, deviceStore.getPorts(DID1).size());
         assertEquals(0, deviceStore.getPorts(DID1).size());
+        verify(clusterCommunicator);
+        // TODO: verify broadcast message
+        assertTrue(bcast.hasCaptured());
 
 
         // putBack Device, Port w/o annotation
         // putBack Device, Port w/o annotation
         putDevice(DID1, SW1);
         putDevice(DID1, SW1);
@@ -563,34 +804,28 @@ public class GossipDeviceStoreTest {
             this.clusterService = clusterService;
             this.clusterService = clusterService;
             this.clusterCommunicator = clusterCommunicator;
             this.clusterCommunicator = clusterCommunicator;
         }
         }
-    }
 
 
-    private static final class TestClusterCommunicationService implements ClusterCommunicationService {
-        @Override
-        public boolean broadcast(ClusterMessage message) throws IOException { return true; }
-        @Override
-        public boolean unicast(ClusterMessage message, NodeId nodeId) throws IOException { return true; }
-        @Override
-        public boolean multicast(ClusterMessage message, Set<NodeId> nodeIds) throws IOException { return true; }
-        @Override
-        public void addSubscriber(MessageSubject subject, ClusterMessageHandler subscriber) {}
+        public <T> T deserialize(byte[] bytes) {
+            return SERIALIZER.decode(bytes);
+        }
     }
     }
 
 
     private static final class TestClusterService implements ClusterService {
     private static final class TestClusterService implements ClusterService {
 
 
-        private static final ControllerNode ONOS1 =
-            new DefaultControllerNode(new NodeId("N1"), IpPrefix.valueOf("127.0.0.1"));
         private final Map<NodeId, ControllerNode> nodes = new HashMap<>();
         private final Map<NodeId, ControllerNode> nodes = new HashMap<>();
         private final Map<NodeId, ControllerNode.State> nodeStates = new HashMap<>();
         private final Map<NodeId, ControllerNode.State> nodeStates = new HashMap<>();
 
 
         public TestClusterService() {
         public TestClusterService() {
-            nodes.put(new NodeId("N1"), ONOS1);
-            nodeStates.put(new NodeId("N1"), ControllerNode.State.ACTIVE);
+            nodes.put(NID1, ONOS1);
+            nodeStates.put(NID1, ACTIVE);
+
+            nodes.put(NID2, ONOS2);
+            nodeStates.put(NID2, ACTIVE);
         }
         }
 
 
         @Override
         @Override
         public ControllerNode getLocalNode() {
         public ControllerNode getLocalNode() {
-            return ONOS1;
+            return GossipDeviceStoreTest.ONOS1;
         }
         }
 
 
         @Override
         @Override

+package org.onlab.onos.store.device.impl.peermsg;
+
+import static org.onlab.onos.net.DeviceId.deviceId;
+
+import org.junit.Test;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.provider.ProviderId;
+
+import com.google.common.testing.EqualsTester;
+
+public class DeviceFragmentIdTest {
+
+    private static final ProviderId PID = new ProviderId("of", "foo");
+    private static final ProviderId PIDA = new ProviderId("of", "bar", true);
+    private static final DeviceId DID1 = deviceId("of:foo");
+    private static final DeviceId DID2 = deviceId("of:bar");
+
+    @Test
+    public final void testEquals() {
+
+        new EqualsTester()
+            .addEqualityGroup(new DeviceFragmentId(DID1, PID),
+                              new DeviceFragmentId(DID1, PID))
+            .addEqualityGroup(new DeviceFragmentId(DID2, PID),
+                              new DeviceFragmentId(DID2, PID))
+            .addEqualityGroup(new DeviceFragmentId(DID1, PIDA),
+                              new DeviceFragmentId(DID1, PIDA))
+            .addEqualityGroup(new DeviceFragmentId(DID2, PIDA),
+                              new DeviceFragmentId(DID2, PIDA))
+        .testEquals();
+    }
+
+}

+package org.onlab.onos.store.device.impl.peermsg;
+
+import static org.onlab.onos.net.DeviceId.deviceId;
+
+import org.junit.Test;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.provider.ProviderId;
+
+import com.google.common.testing.EqualsTester;
+
+public class PortFragmentIdTest {
+
+    private static final ProviderId PID = new ProviderId("of", "foo");
+    private static final ProviderId PIDA = new ProviderId("of", "bar", true);
+
+    private static final DeviceId DID1 = deviceId("of:foo");
+    private static final DeviceId DID2 = deviceId("of:bar");
+
+    private static final PortNumber PN1 = PortNumber.portNumber(1);
+    private static final PortNumber PN2 = PortNumber.portNumber(2);
+
+    @Test
+    public final void testEquals() {
+        new EqualsTester()
+        .addEqualityGroup(new PortFragmentId(DID1, PID, PN1),
+                          new PortFragmentId(DID1, PID, PN1))
+        .addEqualityGroup(new PortFragmentId(DID2, PID, PN1),
+                          new PortFragmentId(DID2, PID, PN1))
+        .addEqualityGroup(new PortFragmentId(DID1, PIDA, PN1),
+                          new PortFragmentId(DID1, PIDA, PN1))
+        .addEqualityGroup(new PortFragmentId(DID2, PIDA, PN1),
+                          new PortFragmentId(DID2, PIDA, PN1))
+
+        .addEqualityGroup(new PortFragmentId(DID1, PID, PN2),
+                          new PortFragmentId(DID1, PID, PN2))
+        .addEqualityGroup(new PortFragmentId(DID2, PID, PN2),
+                          new PortFragmentId(DID2, PID, PN2))
+        .addEqualityGroup(new PortFragmentId(DID1, PIDA, PN2),
+                          new PortFragmentId(DID1, PIDA, PN2))
+        .addEqualityGroup(new PortFragmentId(DID2, PIDA, PN2),
+                          new PortFragmentId(DID2, PIDA, PN2))
+        .testEquals();
+    }
+
+}

@@ -46,10 +46,6 @@
           <groupId>com.hazelcast</groupId>
           <groupId>com.hazelcast</groupId>
           <artifactId>hazelcast</artifactId>
           <artifactId>hazelcast</artifactId>
         </dependency>
         </dependency>
-        <dependency>
-          <groupId>de.javakaffee</groupId>
-          <artifactId>kryo-serializers</artifactId>
-        </dependency>
     </dependencies>
     </dependencies>
 
 
     <build>
     <build>

@@ -35,8 +35,8 @@
           <artifactId>hazelcast</artifactId>
           <artifactId>hazelcast</artifactId>
         </dependency>
         </dependency>
         <dependency>
         <dependency>
-          <groupId>de.javakaffee</groupId>
-          <artifactId>kryo-serializers</artifactId>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
         </dependency>
         </dependency>
     </dependencies>
     </dependencies>
 
 

@@ -23,11 +23,6 @@
         </dependency>
         </dependency>
         <dependency>
         <dependency>
             <groupId>org.onlab.onos</groupId>
             <groupId>org.onlab.onos</groupId>
-            <artifactId>onos-core-serializers</artifactId>
-            <version>${project.version}</version>
-        </dependency>
-        <dependency>
-            <groupId>org.onlab.onos</groupId>
             <artifactId>onos-core-hz-common</artifactId>
             <artifactId>onos-core-hz-common</artifactId>
             <version>${project.version}</version>
             <version>${project.version}</version>
         </dependency>
         </dependency>
@@ -46,10 +41,6 @@
           <groupId>com.hazelcast</groupId>
           <groupId>com.hazelcast</groupId>
           <artifactId>hazelcast</artifactId>
           <artifactId>hazelcast</artifactId>
         </dependency>
         </dependency>
-        <dependency>
-          <groupId>de.javakaffee</groupId>
-          <artifactId>kryo-serializers</artifactId>
-        </dependency>
     </dependencies>
     </dependencies>
 
 
     <build>
     <build>

@@ -43,8 +43,8 @@ public class DistributedFlowRuleStore
     private final Multimap<DeviceId, FlowEntry> flowEntries =
     private final Multimap<DeviceId, FlowEntry> flowEntries =
             ArrayListMultimap.<DeviceId, FlowEntry>create();
             ArrayListMultimap.<DeviceId, FlowEntry>create();
 
 
-    private final Multimap<ApplicationId, FlowRule> flowEntriesById =
-            ArrayListMultimap.<ApplicationId, FlowRule>create();
+    private final Multimap<Short, FlowRule> flowEntriesById =
+            ArrayListMultimap.<Short, FlowRule>create();
 
 
     @Activate
     @Activate
     public void activate() {
     public void activate() {
@@ -83,7 +83,7 @@ public class DistributedFlowRuleStore
 
 
     @Override
     @Override
     public synchronized Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId) {
     public synchronized Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId) {
-        Collection<FlowRule> rules = flowEntriesById.get(appId);
+        Collection<FlowRule> rules = flowEntriesById.get(appId.id());
         if (rules == null) {
         if (rules == null) {
             return Collections.emptyList();
             return Collections.emptyList();
         }
         }

 package org.onlab.onos.store.host.impl;
 package org.onlab.onos.store.host.impl;
 
 
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_ADDED;
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_MOVED;
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_REMOVED;
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_UPDATED;
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.net.Annotations;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.DefaultHost;
 import org.onlab.onos.net.DefaultHost;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.HostId;
 import org.onlab.onos.net.HostId;
+import org.onlab.onos.net.HostLocation;
 import org.onlab.onos.net.host.HostDescription;
 import org.onlab.onos.net.host.HostDescription;
 import org.onlab.onos.net.host.HostEvent;
 import org.onlab.onos.net.host.HostEvent;
 import org.onlab.onos.net.host.HostStore;
 import org.onlab.onos.net.host.HostStore;
@@ -33,10 +27,13 @@ import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.onlab.packet.VlanId;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Sets;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.onlab.onos.net.host.HostEvent.Type.*;
+import static org.slf4j.LoggerFactory.getLogger;
 
 
 /**
 /**
  * TEMPORARY: Manages inventory of end-station hosts using distributed
  * TEMPORARY: Manages inventory of end-station hosts using distributed
@@ -46,13 +43,13 @@ import com.google.common.collect.Sets;
 @Component(immediate = true)
 @Component(immediate = true)
 @Service
 @Service
 public class DistributedHostStore
 public class DistributedHostStore
-extends AbstractStore<HostEvent, HostStoreDelegate>
-implements HostStore {
+        extends AbstractStore<HostEvent, HostStoreDelegate>
+        implements HostStore {
 
 
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
 
 
     // Host inventory
     // Host inventory
-    private final Map<HostId, Host> hosts = new ConcurrentHashMap<>();
+    private final Map<HostId, StoredHost> hosts = new ConcurrentHashMap<>(2000000, 0.75f, 16);
 
 
     // Hosts tracked by their location
     // Hosts tracked by their location
     private final Multimap<ConnectPoint, Host> locations = HashMultimap.create();
     private final Multimap<ConnectPoint, Host> locations = HashMultimap.create();
@@ -72,8 +69,8 @@ implements HostStore {
 
 
     @Override
     @Override
     public HostEvent createOrUpdateHost(ProviderId providerId, HostId hostId,
     public HostEvent createOrUpdateHost(ProviderId providerId, HostId hostId,
-            HostDescription hostDescription) {
-        Host host = hosts.get(hostId);
+                                        HostDescription hostDescription) {
+        StoredHost host = hosts.get(hostId);
         if (host == null) {
         if (host == null) {
             return createHost(providerId, hostId, hostDescription);
             return createHost(providerId, hostId, hostDescription);
         }
         }
@@ -82,12 +79,12 @@ implements HostStore {
 
 
     // creates a new host and sends HOST_ADDED
     // creates a new host and sends HOST_ADDED
     private HostEvent createHost(ProviderId providerId, HostId hostId,
     private HostEvent createHost(ProviderId providerId, HostId hostId,
-            HostDescription descr) {
-        DefaultHost newhost = new DefaultHost(providerId, hostId,
-                descr.hwAddress(),
-                descr.vlan(),
-                descr.location(),
-                descr.ipAddresses());
+                                 HostDescription descr) {
+        StoredHost newhost = new StoredHost(providerId, hostId,
+                                            descr.hwAddress(),
+                                            descr.vlan(),
+                                            descr.location(),
+                                            ImmutableSet.of(descr.ipAddress()));
         synchronized (this) {
         synchronized (this) {
             hosts.put(hostId, newhost);
             hosts.put(hostId, newhost);
             locations.put(descr.location(), newhost);
             locations.put(descr.location(), newhost);
@@ -96,28 +93,24 @@ implements HostStore {
     }
     }
 
 
     // checks for type of update to host, sends appropriate event
     // checks for type of update to host, sends appropriate event
-    private HostEvent updateHost(ProviderId providerId, Host host,
-            HostDescription descr) {
-        DefaultHost updated;
+    private HostEvent updateHost(ProviderId providerId, StoredHost host,
+                                 HostDescription descr) {
         HostEvent event;
         HostEvent event;
         if (!host.location().equals(descr.location())) {
         if (!host.location().equals(descr.location())) {
-            updated = new DefaultHost(providerId, host.id(),
-                    host.mac(),
-                    host.vlan(),
-                    descr.location(),
-                    host.ipAddresses());
-            event = new HostEvent(HOST_MOVED, updated);
-
-        } else if (!(host.ipAddresses().equals(descr.ipAddresses()))) {
-            updated = new DefaultHost(providerId, host.id(),
-                    host.mac(),
-                    host.vlan(),
-                    descr.location(),
-                    descr.ipAddresses());
-            event = new HostEvent(HOST_UPDATED, updated);
-        } else {
+            host.setLocation(descr.location());
+            return new HostEvent(HOST_MOVED, host);
+        }
+
+        if (host.ipAddresses().contains(descr.ipAddress())) {
             return null;
             return null;
         }
         }
+
+        Set<IpPrefix> addresses = new HashSet<>(host.ipAddresses());
+        addresses.add(descr.ipAddress());
+        StoredHost updated = new StoredHost(providerId, host.id(),
+                                            host.mac(), host.vlan(),
+                                            descr.location(), addresses);
+        event = new HostEvent(HOST_UPDATED, updated);
         synchronized (this) {
         synchronized (this) {
             hosts.put(host.id(), updated);
             hosts.put(host.id(), updated);
             locations.remove(host.location(), host);
             locations.remove(host.location(), host);
@@ -145,7 +138,7 @@ implements HostStore {
 
 
     @Override
     @Override
     public Iterable<Host> getHosts() {
     public Iterable<Host> getHosts() {
-        return Collections.unmodifiableSet(new HashSet<>(hosts.values()));
+        return ImmutableSet.<Host>copyOf(hosts.values());
     }
     }
 
 
     @Override
     @Override
@@ -275,4 +268,35 @@ implements HostStore {
         return addresses;
         return addresses;
     }
     }
 
 
+    // Auxiliary extension to allow location to mutate.
+    private class StoredHost extends DefaultHost {
+        private HostLocation location;
+
+        /**
+         * Creates an end-station host using the supplied information.
+         *
+         * @param providerId  provider identity
+         * @param id          host identifier
+         * @param mac         host MAC address
+         * @param vlan        host VLAN identifier
+         * @param location    host location
+         * @param ips         host IP addresses
+         * @param annotations optional key/value annotations
+         */
+        public StoredHost(ProviderId providerId, HostId id,
+                          MacAddress mac, VlanId vlan, HostLocation location,
+                          Set<IpPrefix> ips, Annotations... annotations) {
+            super(providerId, id, mac, vlan, location, ips, annotations);
+            this.location = location;
+        }
+
+        void setLocation(HostLocation location) {
+            this.location = location;
+        }
+
+        @Override
+        public HostLocation location() {
+            return location;
+        }
+    }
 }
 }

@@ -26,8 +26,13 @@
             <artifactId>org.apache.felix.scr.annotations</artifactId>
             <artifactId>org.apache.felix.scr.annotations</artifactId>
         </dependency>
         </dependency>
         <dependency>
         <dependency>
-          <groupId>de.javakaffee</groupId>
-          <artifactId>kryo-serializers</artifactId>
+            <groupId>com.esotericsoftware</groupId>
+            <artifactId>kryo</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.google.guava</groupId>
+            <artifactId>guava-testlib</artifactId>
+            <scope>test</scope>
         </dependency>
         </dependency>
     </dependencies>
     </dependencies>
 
 

+package org.onlab.onos.store.serializers;
+
+import org.onlab.util.KryoPool.FamilySerializer;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableList.Builder;
+
+/**
+ * Creates {@link ImmutableList} serializer instance.
+ */
+public class ImmutableListSerializer extends FamilySerializer<ImmutableList<?>> {
+
+    /**
+     * Creates {@link ImmutableList} serializer instance.
+     */
+    public ImmutableListSerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+    @Override
+    public void write(Kryo kryo, Output output, ImmutableList<?> object) {
+        output.writeInt(object.size());
+        for (Object e : object) {
+            kryo.writeClassAndObject(output, e);
+        }
+    }
+
+    @Override
+    public ImmutableList<?> read(Kryo kryo, Input input,
+            Class<ImmutableList<?>> type) {
+        final int size = input.readInt();
+        Builder<Object> builder = ImmutableList.builder();
+        for (int i = 0; i < size; ++i) {
+            builder.add(kryo.readClassAndObject(input));
+        }
+        return builder.build();
+    }
+
+    @Override
+    public void registerFamilies(Kryo kryo) {
+        kryo.register(ImmutableList.of(1).getClass(), this);
+        kryo.register(ImmutableList.of(1, 2).getClass(), this);
+        // TODO register required ImmutableList variants
+    }
+
+}

@@ -24,12 +24,15 @@ import org.onlab.onos.net.Port;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.device.DefaultDeviceDescription;
 import org.onlab.onos.net.device.DefaultDeviceDescription;
 import org.onlab.onos.net.device.DefaultPortDescription;
 import org.onlab.onos.net.device.DefaultPortDescription;
+import org.onlab.onos.net.link.DefaultLinkDescription;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.Timestamp;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.IpPrefix;
 import org.onlab.util.KryoPool;
 import org.onlab.util.KryoPool;
 
 
-import de.javakaffee.kryoserializers.URISerializer;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
 
 
 public final class KryoPoolUtil {
 public final class KryoPoolUtil {
 
 
@@ -47,23 +50,29 @@ public final class KryoPoolUtil {
      */
      */
     public static final KryoPool API = KryoPool.newBuilder()
     public static final KryoPool API = KryoPool.newBuilder()
             .register(MISC)
             .register(MISC)
+            .register(ImmutableMap.class, new ImmutableMapSerializer())
+            .register(ImmutableList.class, new ImmutableListSerializer())
             .register(
             .register(
                     //
                     //
                     ArrayList.class,
                     ArrayList.class,
                     Arrays.asList().getClass(),
                     Arrays.asList().getClass(),
                     HashMap.class,
                     HashMap.class,
                     //
                     //
+                    //
                     ControllerNode.State.class,
                     ControllerNode.State.class,
                     Device.Type.class,
                     Device.Type.class,
                     DefaultAnnotations.class,
                     DefaultAnnotations.class,
                     DefaultControllerNode.class,
                     DefaultControllerNode.class,
                     DefaultDevice.class,
                     DefaultDevice.class,
                     DefaultDeviceDescription.class,
                     DefaultDeviceDescription.class,
+                    DefaultLinkDescription.class,
                     MastershipRole.class,
                     MastershipRole.class,
                     Port.class,
                     Port.class,
                     DefaultPortDescription.class,
                     DefaultPortDescription.class,
                     Element.class,
                     Element.class,
-                    Link.Type.class
+                    Link.Type.class,
+                    Timestamp.class
+
                     )
                     )
             .register(URI.class, new URISerializer())
             .register(URI.class, new URISerializer())
             .register(NodeId.class, new NodeIdSerializer())
             .register(NodeId.class, new NodeIdSerializer())

 package org.onlab.onos.store.serializers;
 package org.onlab.onos.store.serializers;
 
 
 import org.onlab.util.KryoPool;
 import org.onlab.util.KryoPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
 import java.nio.ByteBuffer;
 import java.nio.ByteBuffer;
 
 
 /**
 /**
@@ -11,10 +8,8 @@ import java.nio.ByteBuffer;
  */
  */
 public class KryoSerializer implements StoreSerializer {
 public class KryoSerializer implements StoreSerializer {
 
 
-    private final Logger log = LoggerFactory.getLogger(getClass());
     protected KryoPool serializerPool;
     protected KryoPool serializerPool;
 
 
-
     public KryoSerializer() {
     public KryoSerializer() {
         setupKryoPool();
         setupKryoPool();
     }
     }

+package org.onlab.onos.store.serializers;
+
+import java.net.URI;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Serializer for {@link URI}.
+ */
+public class URISerializer extends Serializer<URI> {
+
+    /**
+     * Creates {@link URI} serializer instance.
+     */
+    public URISerializer() {
+        super(false);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output, URI object) {
+        output.writeString(object.toString());
+    }
+
+    @Override
+    public URI read(Kryo kryo, Input input, Class<URI> type) {
+        return URI.create(input.readString());
+    }
+}

@@ -42,8 +42,8 @@ public class SimpleFlowRuleStore
     private final Multimap<DeviceId, FlowEntry> flowEntries =
     private final Multimap<DeviceId, FlowEntry> flowEntries =
             ArrayListMultimap.<DeviceId, FlowEntry>create();
             ArrayListMultimap.<DeviceId, FlowEntry>create();
 
 
-    private final Multimap<ApplicationId, FlowRule> flowEntriesById =
-            ArrayListMultimap.<ApplicationId, FlowRule>create();
+    private final Multimap<Short, FlowRule> flowEntriesById =
+            ArrayListMultimap.<Short, FlowRule>create();
 
 
     @Activate
     @Activate
     public void activate() {
     public void activate() {
@@ -82,7 +82,7 @@ public class SimpleFlowRuleStore
 
 
     @Override
     @Override
     public synchronized Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId) {
     public synchronized Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId) {
-        Collection<FlowRule> rules = flowEntriesById.get(appId);
+        Collection<FlowRule> rules = flowEntriesById.get(appId.id());
         if (rules == null) {
         if (rules == null) {
             return Collections.emptyList();
             return Collections.emptyList();
         }
         }

 package org.onlab.onos.store.trivial.impl;
 package org.onlab.onos.store.trivial.impl;
 
 
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_ADDED;
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_MOVED;
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_REMOVED;
-import static org.onlab.onos.net.host.HostEvent.Type.HOST_UPDATED;
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.net.Annotations;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.DefaultHost;
 import org.onlab.onos.net.DefaultHost;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.Host;
 import org.onlab.onos.net.HostId;
 import org.onlab.onos.net.HostId;
+import org.onlab.onos.net.HostLocation;
 import org.onlab.onos.net.host.HostDescription;
 import org.onlab.onos.net.host.HostDescription;
 import org.onlab.onos.net.host.HostEvent;
 import org.onlab.onos.net.host.HostEvent;
 import org.onlab.onos.net.host.HostStore;
 import org.onlab.onos.net.host.HostStore;
@@ -33,10 +27,13 @@ import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.onlab.packet.VlanId;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
-import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Multimap;
-import com.google.common.collect.Sets;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+import static org.onlab.onos.net.host.HostEvent.Type.*;
+import static org.slf4j.LoggerFactory.getLogger;
 
 
 /**
 /**
  * Manages inventory of end-station hosts using trivial in-memory
  * Manages inventory of end-station hosts using trivial in-memory
@@ -51,7 +48,7 @@ public class SimpleHostStore
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
 
 
     // Host inventory
     // Host inventory
-    private final Map<HostId, Host> hosts = new ConcurrentHashMap<>();
+    private final Map<HostId, StoredHost> hosts = new ConcurrentHashMap<>(2000000, 0.75f, 16);
 
 
     // Hosts tracked by their location
     // Hosts tracked by their location
     private final Multimap<ConnectPoint, Host> locations = HashMultimap.create();
     private final Multimap<ConnectPoint, Host> locations = HashMultimap.create();
@@ -72,7 +69,7 @@ public class SimpleHostStore
     @Override
     @Override
     public HostEvent createOrUpdateHost(ProviderId providerId, HostId hostId,
     public HostEvent createOrUpdateHost(ProviderId providerId, HostId hostId,
                                         HostDescription hostDescription) {
                                         HostDescription hostDescription) {
-        Host host = hosts.get(hostId);
+        StoredHost host = hosts.get(hostId);
         if (host == null) {
         if (host == null) {
             return createHost(providerId, hostId, hostDescription);
             return createHost(providerId, hostId, hostDescription);
         }
         }
@@ -82,11 +79,11 @@ public class SimpleHostStore
     // creates a new host and sends HOST_ADDED
     // creates a new host and sends HOST_ADDED
     private HostEvent createHost(ProviderId providerId, HostId hostId,
     private HostEvent createHost(ProviderId providerId, HostId hostId,
                                  HostDescription descr) {
                                  HostDescription descr) {
-        DefaultHost newhost = new DefaultHost(providerId, hostId,
-                                              descr.hwAddress(),
-                                              descr.vlan(),
-                                              descr.location(),
-                                              descr.ipAddresses());
+        StoredHost newhost = new StoredHost(providerId, hostId,
+                                            descr.hwAddress(),
+                                            descr.vlan(),
+                                            descr.location(),
+                                            ImmutableSet.of(descr.ipAddress()));
         synchronized (this) {
         synchronized (this) {
             hosts.put(hostId, newhost);
             hosts.put(hostId, newhost);
             locations.put(descr.location(), newhost);
             locations.put(descr.location(), newhost);
@@ -95,28 +92,24 @@ public class SimpleHostStore
     }
     }
 
 
     // checks for type of update to host, sends appropriate event
     // checks for type of update to host, sends appropriate event
-    private HostEvent updateHost(ProviderId providerId, Host host,
+    private HostEvent updateHost(ProviderId providerId, StoredHost host,
                                  HostDescription descr) {
                                  HostDescription descr) {
-        DefaultHost updated;
         HostEvent event;
         HostEvent event;
         if (!host.location().equals(descr.location())) {
         if (!host.location().equals(descr.location())) {
-            updated = new DefaultHost(providerId, host.id(),
-                                      host.mac(),
-                                      host.vlan(),
-                                      descr.location(),
-                                      host.ipAddresses());
-            event = new HostEvent(HOST_MOVED, updated);
-
-        } else if (!(host.ipAddresses().equals(descr.ipAddresses()))) {
-            updated = new DefaultHost(providerId, host.id(),
-                                      host.mac(),
-                                      host.vlan(),
-                                      descr.location(),
-                                      descr.ipAddresses());
-            event = new HostEvent(HOST_UPDATED, updated);
-        } else {
+            host.setLocation(descr.location());
+            return new HostEvent(HOST_MOVED, host);
+        }
+
+        if (host.ipAddresses().contains(descr.ipAddress())) {
             return null;
             return null;
         }
         }
+
+        Set<IpPrefix> addresses = new HashSet<>(host.ipAddresses());
+        addresses.add(descr.ipAddress());
+        StoredHost updated = new StoredHost(providerId, host.id(),
+                                            host.mac(), host.vlan(),
+                                            descr.location(), addresses);
+        event = new HostEvent(HOST_UPDATED, updated);
         synchronized (this) {
         synchronized (this) {
             hosts.put(host.id(), updated);
             hosts.put(host.id(), updated);
             locations.remove(host.location(), host);
             locations.remove(host.location(), host);
@@ -144,7 +137,7 @@ public class SimpleHostStore
 
 
     @Override
     @Override
     public Iterable<Host> getHosts() {
     public Iterable<Host> getHosts() {
-        return Collections.unmodifiableSet(new HashSet<>(hosts.values()));
+        return ImmutableSet.<Host>copyOf(hosts.values());
     }
     }
 
 
     @Override
     @Override
@@ -274,4 +267,35 @@ public class SimpleHostStore
         return addresses;
         return addresses;
     }
     }
 
 
+    // Auxiliary extension to allow location to mutate.
+    private class StoredHost extends DefaultHost {
+        private HostLocation location;
+
+        /**
+         * Creates an end-station host using the supplied information.
+         *
+         * @param providerId  provider identity
+         * @param id          host identifier
+         * @param mac         host MAC address
+         * @param vlan        host VLAN identifier
+         * @param location    host location
+         * @param ips         host IP addresses
+         * @param annotations optional key/value annotations
+         */
+        public StoredHost(ProviderId providerId, HostId id,
+                          MacAddress mac, VlanId vlan, HostLocation location,
+                          Set<IpPrefix> ips, Annotations... annotations) {
+            super(providerId, id, mac, vlan, location, ips, annotations);
+            this.location = location;
+        }
+
+        void setLocation(HostLocation location) {
+            this.location = location;
+        }
+
+        @Override
+        public HostLocation location() {
+            return location;
+        }
+    }
 }
 }

@@ -20,10 +20,11 @@
         <bundle>mvn:io.dropwizard.metrics/metrics-core/3.1.0</bundle>
         <bundle>mvn:io.dropwizard.metrics/metrics-core/3.1.0</bundle>
         <bundle>mvn:com.eclipsesource.minimal-json/minimal-json/0.9.1</bundle>
         <bundle>mvn:com.eclipsesource.minimal-json/minimal-json/0.9.1</bundle>
 
 
-        <bundle>mvn:com.esotericsoftware.kryo/kryo/2.24.0</bundle>
+        <bundle>mvn:com.esotericsoftware/kryo/3.0.0</bundle>
+        <bundle>mvn:com.esotericsoftware/reflectasm/1.10.0</bundle>
+        <bundle>mvn:org.ow2.asm/asm/4.2</bundle>
         <bundle>mvn:com.esotericsoftware/minlog/1.3.0</bundle>
         <bundle>mvn:com.esotericsoftware/minlog/1.3.0</bundle>
         <bundle>mvn:org.objenesis/objenesis/2.1</bundle>
         <bundle>mvn:org.objenesis/objenesis/2.1</bundle>
-        <bundle>mvn:de.javakaffee/kryo-serializers/0.27</bundle>
 
 
         <bundle>mvn:org.onlab.onos/onlab-nio/1.0.0-SNAPSHOT</bundle>
         <bundle>mvn:org.onlab.onos/onlab-nio/1.0.0-SNAPSHOT</bundle>
 
 

@@ -30,7 +30,7 @@
             <groupId>org.projectfloodlight</groupId>
             <groupId>org.projectfloodlight</groupId>
             <artifactId>openflowj</artifactId>
             <artifactId>openflowj</artifactId>
             <!-- FIXME once experimenter gets merged to upstream -->
             <!-- FIXME once experimenter gets merged to upstream -->
-            <version>0.3.8-optical_experimenter</version>
+            <version>0.3.8-optical_experimenter2</version>
         </dependency>
         </dependency>
         <dependency>
         <dependency>
             <groupId>io.netty</groupId>
             <groupId>io.netty</groupId>

 package org.onlab.onos.openflow.controller;
 package org.onlab.onos.openflow.controller;
 
 
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.Collections;
-import java.util.concurrent.atomic.AtomicBoolean;
-
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ethernet;
 import org.projectfloodlight.openflow.protocol.OFPacketIn;
 import org.projectfloodlight.openflow.protocol.OFPacketIn;
 import org.projectfloodlight.openflow.protocol.OFPacketOut;
 import org.projectfloodlight.openflow.protocol.OFPacketOut;
@@ -13,11 +8,11 @@ import org.projectfloodlight.openflow.protocol.action.OFActionOutput;
 import org.projectfloodlight.openflow.protocol.match.MatchField;
 import org.projectfloodlight.openflow.protocol.match.MatchField;
 import org.projectfloodlight.openflow.types.OFBufferId;
 import org.projectfloodlight.openflow.types.OFBufferId;
 import org.projectfloodlight.openflow.types.OFPort;
 import org.projectfloodlight.openflow.types.OFPort;
-import org.slf4j.Logger;
 
 
-public final class DefaultOpenFlowPacketContext implements OpenFlowPacketContext {
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 
-    private final Logger log = getLogger(getClass());
+public final class DefaultOpenFlowPacketContext implements OpenFlowPacketContext {
 
 
     private final AtomicBoolean free = new AtomicBoolean(true);
     private final AtomicBoolean free = new AtomicBoolean(true);
     private final AtomicBoolean isBuilt = new AtomicBoolean(false);
     private final AtomicBoolean isBuilt = new AtomicBoolean(false);
@@ -82,7 +77,7 @@ public final class DefaultOpenFlowPacketContext implements OpenFlowPacketContext
     }
     }
 
 
     public static OpenFlowPacketContext packetContextFromPacketIn(OpenFlowSwitch s,
     public static OpenFlowPacketContext packetContextFromPacketIn(OpenFlowSwitch s,
-            OFPacketIn pkt) {
+                                                                  OFPacketIn pkt) {
         return new DefaultOpenFlowPacketContext(s, pkt);
         return new DefaultOpenFlowPacketContext(s, pkt);
     }
     }
 
 

@@ -157,9 +157,7 @@ public class Controller {
         }
         }
         log.debug("OpenFlow port set to {}", this.openFlowPort);
         log.debug("OpenFlow port set to {}", this.openFlowPort);
         String threads = configParams.get("workerthreads");
         String threads = configParams.get("workerthreads");
-        if (threads != null) {
-            this.workerThreads = Integer.parseInt(threads);
-        }
+        this.workerThreads = threads != null ? Integer.parseInt(threads) : 16;
         log.debug("Number of worker threads set to {}", this.workerThreads);
         log.debug("Number of worker threads set to {}", this.workerThreads);
     }
     }
 
 

@@ -981,13 +981,13 @@ class OFChannelHandler extends IdleStateAwareChannelHandler {
                 // switch was a duplicate-dpid, calling the method below would clear
                 // switch was a duplicate-dpid, calling the method below would clear
                 // all state for the original switch (with the same dpid),
                 // all state for the original switch (with the same dpid),
                 // which we obviously don't want.
                 // which we obviously don't want.
-                log.info("{}:removal called");
+                log.info("{}:removal called", getSwitchInfoString());
                 sw.removeConnectedSwitch();
                 sw.removeConnectedSwitch();
             } else {
             } else {
                 // A duplicate was disconnected on this ChannelHandler,
                 // A duplicate was disconnected on this ChannelHandler,
                 // this is the same switch reconnecting, but the original state was
                 // this is the same switch reconnecting, but the original state was
                 // not cleaned up - XXX check liveness of original ChannelHandler
                 // not cleaned up - XXX check liveness of original ChannelHandler
-                log.info("{}:duplicate found");
+                log.info("{}:duplicate found", getSwitchInfoString());
                 duplicateDpidFound = Boolean.FALSE;
                 duplicateDpidFound = Boolean.FALSE;
             }
             }
         } else {
         } else {

@@ -44,7 +44,6 @@ public class OpenFlowControllerImpl implements OpenFlowController {
     private final ExecutorService executor = Executors.newFixedThreadPool(16,
     private final ExecutorService executor = Executors.newFixedThreadPool(16,
             namedThreads("of-event-%d"));
             namedThreads("of-event-%d"));
 
 
-
     protected ConcurrentHashMap<Dpid, OpenFlowSwitch> connectedSwitches =
     protected ConcurrentHashMap<Dpid, OpenFlowSwitch> connectedSwitches =
             new ConcurrentHashMap<Dpid, OpenFlowSwitch>();
             new ConcurrentHashMap<Dpid, OpenFlowSwitch>();
     protected ConcurrentHashMap<Dpid, OpenFlowSwitch> activeMasterSwitches =
     protected ConcurrentHashMap<Dpid, OpenFlowSwitch> activeMasterSwitches =

@@ -57,6 +57,12 @@ public final class DriverManager implements OpenFlowSwitchDriverFactory {
             }
             }
         }
         }
 
 
+        String sw = desc.getSwDesc();
+        if (sw.startsWith("LINC-OE")) {
+            log.debug("Optical Emulator LINC-OE with DPID:{} found..", dpid);
+            return new OFOpticalSwitchImplLINC13(dpid, desc);
+        }
+
         log.warn("DriverManager could not identify switch desc: {}. "
         log.warn("DriverManager could not identify switch desc: {}. "
                 + "Assigning AbstractOpenFlowSwich", desc);
                 + "Assigning AbstractOpenFlowSwich", desc);
         return new AbstractOpenFlowSwitch(dpid, desc) {
         return new AbstractOpenFlowSwitch(dpid, desc) {

+package org.onlab.onos.openflow.drivers.impl;
+
+import org.onlab.onos.openflow.controller.driver.SwitchDriverSubHandshakeAlreadyStarted;
+import org.onlab.onos.openflow.controller.driver.SwitchDriverSubHandshakeCompleted;
+import org.onlab.onos.openflow.controller.driver.SwitchDriverSubHandshakeNotStarted;
+import org.onlab.onos.openflow.controller.Dpid;
+import org.onlab.onos.openflow.controller.driver.AbstractOpenFlowSwitch;
+import org.projectfloodlight.openflow.protocol.OFCircuitPortsReply;
+import org.projectfloodlight.openflow.protocol.OFCircuitPortsRequest;
+import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFErrorMsg;
+import org.projectfloodlight.openflow.protocol.OFMatchV3;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFOxmList;
+import org.projectfloodlight.openflow.protocol.OFPortDesc;
+import org.projectfloodlight.openflow.protocol.OFPortOptical;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+import org.projectfloodlight.openflow.protocol.action.OFActionCircuit;
+import org.projectfloodlight.openflow.protocol.instruction.OFInstruction;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmInPort;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmOchSigid;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmOchSigidBasic;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmOchSigtype;
+import org.projectfloodlight.openflow.protocol.oxm.OFOxmOchSigtypeBasic;
+import org.projectfloodlight.openflow.types.CircuitSignalID;
+import org.projectfloodlight.openflow.types.OFPort;
+import org.projectfloodlight.openflow.types.U8;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * LINC-OE Optical Emulator switch class.
+ */
+public class OFOpticalSwitchImplLINC13 extends AbstractOpenFlowSwitch {
+
+    private final AtomicBoolean driverHandshakeComplete;
+    private long barrierXidToWaitFor = -1;
+
+    private final Logger log =
+            LoggerFactory.getLogger(OFOpticalSwitchImplLINC13.class);
+
+    OFOpticalSwitchImplLINC13(Dpid dpid, OFDescStatsReply desc) {
+        super(dpid);
+        //setAttribute("optical", "true");
+        driverHandshakeComplete = new AtomicBoolean(false);
+        setSwitchDescription(desc);
+    }
+
+    @Override
+    public String toString() {
+        return "OFOpticalSwitchImplLINC13 [" + ((channel != null)
+                ? channel.getRemoteAddress() : "?")
+                + " DPID[" + ((getStringId() != null) ? getStringId() : "?") + "]]";
+    }
+
+    @Override
+    public void startDriverHandshake() {
+        log.debug("Starting driver handshake for sw {}", getStringId());
+        if (startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeAlreadyStarted();
+        }
+        startDriverHandshakeCalled = true;
+        try {
+            sendHandshakeOFExperimenterPortDescRequest();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public boolean isDriverHandshakeComplete() {
+        if (!startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeNotStarted();
+        }
+        return driverHandshakeComplete.get();
+    }
+
+    @Override
+    public void processDriverHandshakeMessage(OFMessage m) {
+        if (!startDriverHandshakeCalled) {
+            throw new SwitchDriverSubHandshakeNotStarted();
+        }
+        if (driverHandshakeComplete.get()) {
+            throw new SwitchDriverSubHandshakeCompleted(m);
+        }
+
+        switch (m.getType()) {
+            case BARRIER_REPLY:
+                if (m.getXid() == barrierXidToWaitFor) {
+                    log.debug("LINC-OE Received barrier response");
+                }
+                break;
+            case ERROR:
+                log.error("Switch {} Error {}", getStringId(), (OFErrorMsg) m);
+                break;
+            case FEATURES_REPLY:
+                break;
+            case FLOW_REMOVED:
+                break;
+            case GET_ASYNC_REPLY:
+                break;
+            case PACKET_IN:
+                break;
+            case PORT_STATUS:
+                break;
+            case QUEUE_GET_CONFIG_REPLY:
+                break;
+            case ROLE_REPLY:
+                break;
+            case STATS_REPLY:
+                log.debug("LINC-OE : Received stats reply message {}", m);
+                processHandshakeOFExperimenterPortDescRequest(
+                        (OFCircuitPortsReply) m);
+                driverHandshakeComplete.set(true);
+               /* try {
+                    testMA();
+                } catch (IOException e) {
+                    e.printStackTrace();
+                }*/
+                break;
+            default:
+                log.debug("Received message {} during switch-driver " +
+                                  "subhandshake " + "from switch {} ... " +
+                                  "Ignoring message", m,
+                          getStringId());
+
+        }
+    }
+
+
+    private void processHandshakeOFExperimenterPortDescRequest(
+            OFCircuitPortsReply sr) {
+        Collection<OFPortOptical> entries = sr.getEntries();
+        List<OFPortDesc> ofPortDescList = new ArrayList<>(entries.size());
+        for (OFPortOptical entry : entries) {
+            ofPortDescList.add(factory().buildPortDesc().
+                    setPortNo(entry.getPortNo())
+                                           .setConfig(entry.getConfig())
+                                           .setState(entry.getState())
+                                           .setHwAddr(entry.getHwAddr())
+                                           .setName(entry.getName())
+                                           .build());
+        }
+        setPortDescReply(factory().buildPortDescStatsReply().
+                setEntries(ofPortDescList).build());
+    }
+
+
+    private void sendHandshakeOFExperimenterPortDescRequest() throws
+            IOException {
+        // send multi part message for port description for optical switches
+        OFCircuitPortsRequest circuitPortsRequest = factory()
+                .buildCircuitPortsRequest().setXid(getNextTransactionId())
+                .build();
+        log.debug("LINC-OE : Sending experimented circuit port stats " +
+                          "message " +
+                          "{}",
+                  circuitPortsRequest.toString());
+        channel.write(Collections.singletonList(circuitPortsRequest));
+    }
+
+
+
+    //todo for testing
+    public static final U8 SIGNAL_TYPE = U8.of((short) 1);
+    private void testMA() throws IOException {
+        log.debug("LINC OE *** Testing MA ");
+        short lambda = 100;
+        if (getId() == 0x0000ffffffffff02L) {
+            final int inport = 10;
+            final int outport = 20;
+            //Circuit signal id
+            CircuitSignalID sigID = getSignalID(lambda);
+
+            OFOxmOchSigid fieldSigIDMatch = factory().oxms().ochSigid(sigID);
+            OFOxmOchSigtype fieldSigType = factory()
+                    .oxms()
+                    .ochSigtype(SIGNAL_TYPE);
+
+            OFOxmOchSigidBasic ofOxmOchSigidBasic =
+                    factory().oxms().ochSigidBasic(sigID);
+
+            OFOxmOchSigtypeBasic ofOxmOchSigtypeBasic =
+                    factory().oxms().ochSigtypeBasic(SIGNAL_TYPE);
+
+            //Match Port
+            OFOxmInPort fieldPort = factory().oxms()
+                                                .inPort(OFPort.of(inport));
+            OFMatchV3 matchPort =
+                    factory()
+                            .buildMatchV3().
+                            setOxmList(OFOxmList.of(fieldPort,
+                                                    fieldSigType,
+                                                    fieldSigIDMatch)).build();
+
+
+            // Set Action outport ,sigType and sigID
+            List<OFAction> actionList = new ArrayList<>();
+            OFAction actionOutPort =
+                    factory().actions().output(OFPort.of(outport),
+                                                  Short.MAX_VALUE);
+
+            OFActionCircuit actionCircuit = factory()
+                    .actions()
+                    .circuit(ofOxmOchSigidBasic);
+            OFActionCircuit setActionSigType = factory()
+                    .actions()
+                    .circuit(ofOxmOchSigtypeBasic);
+
+            actionList.add(actionOutPort);
+            actionList.add(setActionSigType);
+            actionList.add(actionCircuit);
+
+            OFInstruction instructionAction =
+                    factory().instructions().buildApplyActions()
+                                .setActions(actionList)
+                                .build();
+            List<OFInstruction> instructions =
+                    Collections.singletonList(instructionAction);
+
+            OFMessage opticalFlowEntry =
+                    factory().buildFlowAdd()
+                                .setMatch(matchPort)
+                                .setInstructions(instructions)
+                                .setXid(getNextTransactionId())
+                                .build();
+            log.debug("Adding optical flow in sw {}", getStringId());
+            List<OFMessage> msglist = new ArrayList<>(1);
+            msglist.add(opticalFlowEntry);
+            write(msglist);
+        } else if (getId() == 0x0000ffffffffff03L) {
+            final int inport = 21;
+            final int outport = 22;
+            //Circuit signal id
+            CircuitSignalID sigID = getSignalID(lambda);
+
+            OFOxmOchSigid fieldSigIDMatch = factory().oxms().ochSigid(sigID);
+            OFOxmOchSigtype fieldSigType = factory()
+                    .oxms()
+                    .ochSigtype(SIGNAL_TYPE);
+
+            OFOxmOchSigidBasic ofOxmOchSigidBasic =
+                    factory().oxms().ochSigidBasic(sigID);
+
+            OFOxmOchSigtypeBasic ofOxmOchSigtypeBasic =
+                    factory().oxms().ochSigtypeBasic(SIGNAL_TYPE);
+
+            //Match Port,SigType,SigID
+            OFOxmInPort fieldPort = factory()
+                    .oxms()
+                    .inPort(OFPort.of(inport));
+            OFMatchV3 matchPort = factory()
+                    .buildMatchV3()
+                    .setOxmList(OFOxmList.of(fieldPort,
+                                             fieldSigType,
+                                             fieldSigIDMatch))
+                    .build();
+
+            // Set Action outport ,SigType, sigID
+            List<OFAction> actionList = new ArrayList<>();
+            OFAction actionOutPort =
+                    factory().actions().output(OFPort.of(outport),
+                                                  Short.MAX_VALUE);
+
+            OFActionCircuit setActionSigType = factory()
+                    .actions()
+                    .circuit(ofOxmOchSigtypeBasic);
+            OFActionCircuit actionCircuit = factory()
+                    .actions()
+                    .circuit(ofOxmOchSigidBasic);
+
+
+            actionList.add(actionOutPort);
+            actionList.add(setActionSigType);
+            actionList.add(actionCircuit);
+
+            OFInstruction instructionAction =
+                    factory().instructions().buildApplyActions()
+                                .setActions(actionList)
+                                .build();
+            List<OFInstruction> instructions =
+                    Collections.singletonList(instructionAction);
+
+            OFMessage opticalFlowEntry =
+                    factory().buildFlowAdd()
+                                 .setMatch(matchPort)
+                                 .setInstructions(instructions)
+                                 .setXid(getNextTransactionId())
+                                 .build();
+            log.debug("Adding optical flow in sw {}", getStringId());
+            List<OFMessage> msglist = new ArrayList<>(1);
+            msglist.add(opticalFlowEntry);
+            write(msglist);
+
+        } else if (getId() == 0x0000ffffffffff04L) {
+            final int inport = 23;
+            final int outport = 11;
+            //Circuit signal id
+            CircuitSignalID sigID = getSignalID(lambda);
+
+            OFOxmOchSigid fieldSigIDMatch = factory().oxms().ochSigid(sigID);
+            OFOxmOchSigtype fieldSigType = factory()
+                    .oxms()
+                    .ochSigtype(SIGNAL_TYPE);
+
+
+            //Match Port, sig type and sig id
+            OFOxmInPort fieldPort = factory()
+                    .oxms()
+                    .inPort(OFPort.of(inport));
+            OFMatchV3 matchPort =
+                    factory().buildMatchV3()
+                                .setOxmList(OFOxmList.of(fieldPort,
+                                                         fieldSigType,
+                                                         fieldSigIDMatch))
+                                .build();
+
+            // Set Action outport
+            List<OFAction> actionList = new ArrayList<>();
+            OFAction actionOutPort =
+                    factory().actions().output(OFPort.of(outport),
+                                                  Short.MAX_VALUE);
+
+            actionList.add(actionOutPort);
+
+            OFInstruction instructionAction =
+                    factory().instructions().buildApplyActions()
+                                .setActions(actionList)
+                                .build();
+            List<OFInstruction> instructions =
+                    Collections.singletonList(instructionAction);
+
+            OFMessage opticalFlowEntry =
+                    factory().buildFlowAdd()
+                                 .setMatch(matchPort)
+                                 .setInstructions(instructions)
+                                 .setXid(getNextTransactionId())
+                                 .build();
+            log.debug("Adding optical flow in sw {}", getStringId());
+            List<OFMessage> msglist = new ArrayList<>(1);
+            msglist.add(opticalFlowEntry);
+            write(msglist);
+        }
+
+    }
+
+    // Todo remove - for testing purpose only
+    private static CircuitSignalID getSignalID(short lambda) {
+        byte myGrid = 1;
+        byte myCs = 2;
+        short myCn = lambda;
+        short mySw = 1;
+
+        CircuitSignalID signalID = new CircuitSignalID(myGrid,
+                                                       myCs,
+                                                       myCn,
+                                                       mySw);
+        return signalID;
+    }
+
+    @Override
+    public void write(OFMessage msg) {
+        this.channel.write(msg);
+    }
+
+    @Override
+    public void write(List<OFMessage> msgs) {
+        this.channel.write(msgs);
+    }
+
+    @Override
+    public Boolean supportNxRole() {
+        return false;
+    }
+
+}

@@ -193,9 +193,20 @@
                 <version>0.9.1</version>
                 <version>0.9.1</version>
             </dependency>
             </dependency>
             <dependency>
             <dependency>
-              <groupId>com.esotericsoftware.kryo</groupId>
-              <artifactId>kryo</artifactId>
-              <version>2.24.0</version>
+                <groupId>com.esotericsoftware</groupId>
+                <artifactId>kryo</artifactId>
+                <version>3.0.0</version>
+            </dependency>
+            <dependency>
+                <groupId>com.esotericsoftware</groupId>
+                <artifactId>reflectasm</artifactId>
+                <version>1.10.0</version>
+                <type>bundle</type>
+            </dependency>
+            <dependency>
+                <groupId>org.ow2.asm</groupId>
+                <artifactId>asm</artifactId>
+                <version>4.2</version>
             </dependency>
             </dependency>
             <dependency>
             <dependency>
                 <groupId>com.esotericsoftware</groupId>
                 <groupId>com.esotericsoftware</groupId>
@@ -207,11 +218,6 @@
                 <artifactId>objenesis</artifactId>
                 <artifactId>objenesis</artifactId>
                 <version>2.1</version>
                 <version>2.1</version>
             </dependency>
             </dependency>
-            <dependency>
-              <groupId>de.javakaffee</groupId>
-              <artifactId>kryo-serializers</artifactId>
-              <version>0.27</version>
-            </dependency>
 
 
             <!-- ONOS related -->
             <!-- ONOS related -->
             <dependency>
             <dependency>
@@ -284,6 +290,10 @@
         </dependency>
         </dependency>
         <dependency>
         <dependency>
             <groupId>org.slf4j</groupId>
             <groupId>org.slf4j</groupId>
+            <artifactId>slf4j-api</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.slf4j</groupId>
             <artifactId>slf4j-jdk14</artifactId>
             <artifactId>slf4j-jdk14</artifactId>
         </dependency>
         </dependency>
     </dependencies>
     </dependencies>
@@ -434,9 +444,6 @@
                 <version>3.2</version>
                 <version>3.2</version>
                 <configuration>
                 <configuration>
                     <excludes>
                     <excludes>
-                        <exclude>**/datastore/serializers/**</exclude>
-                        <exclude>**/edu/stanford/**</exclude>
-                        <exclude>**/net/floodlightcontroller/**</exclude>
                     </excludes>
                     </excludes>
                     <rulesets>
                     <rulesets>
                         <ruleset>onos/pmd.xml</ruleset>
                         <ruleset>onos/pmd.xml</ruleset>
@@ -493,7 +500,7 @@
                         <group>
                         <group>
                             <title>Core Subsystems</title>
                             <title>Core Subsystems</title>
                             <packages>
                             <packages>
-                                org.onlab.onos.cluster.impl:org.onlab.onos.net.device.impl:org.onlab.onos.net.link.impl:org.onlab.onos.net.host.impl:org.onlab.onos.net.topology.impl:org.onlab.onos.net.packet.impl:org.onlab.onos.net.flow.impl:org.onlab.onos.store.trivial.*:org.onlab.onos.net.*.impl:org.onlab.onos.event.impl:org.onlab.onos.store.*:org.onlab.onos.net.intent.impl
+                                org.onlab.onos.impl:org.onlab.onos.cluster.impl:org.onlab.onos.net.device.impl:org.onlab.onos.net.link.impl:org.onlab.onos.net.host.impl:org.onlab.onos.net.topology.impl:org.onlab.onos.net.packet.impl:org.onlab.onos.net.flow.impl:org.onlab.onos.store.trivial.*:org.onlab.onos.net.*.impl:org.onlab.onos.event.impl:org.onlab.onos.store.*:org.onlab.onos.net.intent.impl:org.onlab.onos.net.proxyarp.impl
                             </packages>
                             </packages>
                         </group>
                         </group>
                         <group>
                         <group>
@@ -518,7 +525,7 @@
                         <group>
                         <group>
                             <title>Sample Applications</title>
                             <title>Sample Applications</title>
                             <packages>
                             <packages>
-                                org.onlab.onos.tvue:org.onlab.onos.fwd:org.onlab.onos.foo
+                                org.onlab.onos.tvue:org.onlab.onos.fwd:org.onlab.onos.ifwd:org.onlab.onos.mobility:org.onlab.onos.proxyarp:org.onlab.onos.foo
                             </packages>
                             </packages>
                         </group>
                         </group>
                     </groups>
                     </groups>
@@ -545,9 +552,6 @@
                 <version>3.2</version>
                 <version>3.2</version>
                 <configuration>
                 <configuration>
                     <excludes>
                     <excludes>
-                        <exclude>**/datastore/serializers/**</exclude>
-                        <exclude>**/edu/stanford/**</exclude>
-                        <exclude>**/net/floodlightcontroller/**</exclude>
                     </excludes>
                     </excludes>
                     <rulesets>
                     <rulesets>
                         <ruleset>onos/pmd.xml</ruleset>
                         <ruleset>onos/pmd.xml</ruleset>

@@ -27,6 +27,8 @@ import org.onlab.onos.net.flow.instructions.L2ModificationInstruction.ModVlanPcp
 import org.onlab.onos.net.flow.instructions.L3ModificationInstruction;
 import org.onlab.onos.net.flow.instructions.L3ModificationInstruction;
 import org.onlab.onos.net.flow.instructions.L3ModificationInstruction.ModIPInstruction;
 import org.onlab.onos.net.flow.instructions.L3ModificationInstruction.ModIPInstruction;
 import org.projectfloodlight.openflow.protocol.OFFactory;
 import org.projectfloodlight.openflow.protocol.OFFactory;
+import org.projectfloodlight.openflow.protocol.OFFlowAdd;
+import org.projectfloodlight.openflow.protocol.OFFlowDelete;
 import org.projectfloodlight.openflow.protocol.OFFlowMod;
 import org.projectfloodlight.openflow.protocol.OFFlowMod;
 import org.projectfloodlight.openflow.protocol.OFFlowModFlags;
 import org.projectfloodlight.openflow.protocol.OFFlowModFlags;
 import org.projectfloodlight.openflow.protocol.action.OFAction;
 import org.projectfloodlight.openflow.protocol.action.OFAction;
@@ -68,12 +70,13 @@ public class FlowModBuilder {
         this.cookie = flowRule.id();
         this.cookie = flowRule.id();
     }
     }
 
 
-    public OFFlowMod buildFlowAdd() {
+    public OFFlowAdd buildFlowAdd() {
         Match match = buildMatch();
         Match match = buildMatch();
         List<OFAction> actions = buildActions();
         List<OFAction> actions = buildActions();
 
 
         //TODO: what to do without bufferid? do we assume that there will be a pktout as well?
         //TODO: what to do without bufferid? do we assume that there will be a pktout as well?
-        OFFlowMod fm = factory.buildFlowAdd()
+        OFFlowAdd fm = factory.buildFlowAdd()
+                .setXid(cookie.value())
                 .setCookie(U64.of(cookie.value()))
                 .setCookie(U64.of(cookie.value()))
                 .setBufferId(OFBufferId.NO_BUFFER)
                 .setBufferId(OFBufferId.NO_BUFFER)
                 .setActions(actions)
                 .setActions(actions)
@@ -92,6 +95,7 @@ public class FlowModBuilder {
 
 
         //TODO: what to do without bufferid? do we assume that there will be a pktout as well?
         //TODO: what to do without bufferid? do we assume that there will be a pktout as well?
         OFFlowMod fm = factory.buildFlowModify()
         OFFlowMod fm = factory.buildFlowModify()
+                .setXid(cookie.value())
                 .setCookie(U64.of(cookie.value()))
                 .setCookie(U64.of(cookie.value()))
                 .setBufferId(OFBufferId.NO_BUFFER)
                 .setBufferId(OFBufferId.NO_BUFFER)
                 .setActions(actions)
                 .setActions(actions)
@@ -104,11 +108,12 @@ public class FlowModBuilder {
 
 
     }
     }
 
 
-    public OFFlowMod buildFlowDel() {
+    public OFFlowDelete buildFlowDel() {
         Match match = buildMatch();
         Match match = buildMatch();
         List<OFAction> actions = buildActions();
         List<OFAction> actions = buildActions();
 
 
-        OFFlowMod fm = factory.buildFlowDelete()
+        OFFlowDelete fm = factory.buildFlowDelete()
+                .setXid(cookie.value())
                 .setCookie(U64.of(cookie.value()))
                 .setCookie(U64.of(cookie.value()))
                 .setBufferId(OFBufferId.NO_BUFFER)
                 .setBufferId(OFBufferId.NO_BUFFER)
                 .setActions(actions)
                 .setActions(actions)

@@ -2,6 +2,7 @@ package org.onlab.onos.provider.of.flow.impl;
 
 
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
 
 
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.HashSet;
 import java.util.List;
 import java.util.List;
 import java.util.Map;
 import java.util.Map;
@@ -21,9 +22,12 @@ import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.flow.CompletedBatchOperation;
+import org.onlab.onos.net.flow.DefaultFlowEntry;
 import org.onlab.onos.net.flow.FlowEntry;
 import org.onlab.onos.net.flow.FlowEntry;
 import org.onlab.onos.net.flow.FlowRule;
 import org.onlab.onos.net.flow.FlowRule;
 import org.onlab.onos.net.flow.FlowRuleBatchEntry;
 import org.onlab.onos.net.flow.FlowRuleBatchEntry;
+import org.onlab.onos.net.flow.FlowRuleBatchEntry.FlowRuleOperation;
 import org.onlab.onos.net.flow.FlowRuleProvider;
 import org.onlab.onos.net.flow.FlowRuleProvider;
 import org.onlab.onos.net.flow.FlowRuleProviderRegistry;
 import org.onlab.onos.net.flow.FlowRuleProviderRegistry;
 import org.onlab.onos.net.flow.FlowRuleProviderService;
 import org.onlab.onos.net.flow.FlowRuleProviderService;
@@ -40,6 +44,7 @@ import org.onlab.onos.openflow.controller.RoleState;
 import org.projectfloodlight.openflow.protocol.OFActionType;
 import org.projectfloodlight.openflow.protocol.OFActionType;
 import org.projectfloodlight.openflow.protocol.OFBarrierRequest;
 import org.projectfloodlight.openflow.protocol.OFBarrierRequest;
 import org.projectfloodlight.openflow.protocol.OFErrorMsg;
 import org.projectfloodlight.openflow.protocol.OFErrorMsg;
+import org.projectfloodlight.openflow.protocol.OFFlowMod;
 import org.projectfloodlight.openflow.protocol.OFFlowRemoved;
 import org.projectfloodlight.openflow.protocol.OFFlowRemoved;
 import org.projectfloodlight.openflow.protocol.OFFlowStatsEntry;
 import org.projectfloodlight.openflow.protocol.OFFlowStatsEntry;
 import org.projectfloodlight.openflow.protocol.OFFlowStatsReply;
 import org.projectfloodlight.openflow.protocol.OFFlowStatsReply;
@@ -52,6 +57,11 @@ import org.projectfloodlight.openflow.protocol.OFStatsType;
 import org.projectfloodlight.openflow.protocol.OFVersion;
 import org.projectfloodlight.openflow.protocol.OFVersion;
 import org.projectfloodlight.openflow.protocol.action.OFAction;
 import org.projectfloodlight.openflow.protocol.action.OFAction;
 import org.projectfloodlight.openflow.protocol.action.OFActionOutput;
 import org.projectfloodlight.openflow.protocol.action.OFActionOutput;
+import org.projectfloodlight.openflow.protocol.errormsg.OFBadActionErrorMsg;
+import org.projectfloodlight.openflow.protocol.errormsg.OFBadInstructionErrorMsg;
+import org.projectfloodlight.openflow.protocol.errormsg.OFBadMatchErrorMsg;
+import org.projectfloodlight.openflow.protocol.errormsg.OFBadRequestErrorMsg;
+import org.projectfloodlight.openflow.protocol.errormsg.OFFlowModFailedErrorMsg;
 import org.projectfloodlight.openflow.protocol.instruction.OFInstruction;
 import org.projectfloodlight.openflow.protocol.instruction.OFInstruction;
 import org.projectfloodlight.openflow.protocol.instruction.OFInstructionApplyActions;
 import org.projectfloodlight.openflow.protocol.instruction.OFInstructionApplyActions;
 import org.projectfloodlight.openflow.types.OFPort;
 import org.projectfloodlight.openflow.types.OFPort;
@@ -70,6 +80,8 @@ import com.google.common.collect.Multimap;
 @Component(immediate = true)
 @Component(immediate = true)
 public class OpenFlowRuleProvider extends AbstractProvider implements FlowRuleProvider {
 public class OpenFlowRuleProvider extends AbstractProvider implements FlowRuleProvider {
 
 
+    enum BatchState { STARTED, FINISHED, CANCELLED };
+
     private final Logger log = getLogger(getClass());
     private final Logger log = getLogger(getClass());
 
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
@@ -88,6 +100,9 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
     private final Map<Long, InstallationFuture> pendingFutures =
     private final Map<Long, InstallationFuture> pendingFutures =
             new ConcurrentHashMap<Long, InstallationFuture>();
             new ConcurrentHashMap<Long, InstallationFuture>();
 
 
+    private final Map<Long, InstallationFuture> pendingFMs =
+            new ConcurrentHashMap<Long, InstallationFuture>();
+
     /**
     /**
      * Creates an OpenFlow host provider.
      * Creates an OpenFlow host provider.
      */
      */
@@ -143,9 +158,47 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
         removeFlowRule(flowRules);
         removeFlowRule(flowRules);
     }
     }
 
 
+    @Override
+    public Future<CompletedBatchOperation> executeBatch(BatchOperation<FlowRuleBatchEntry> batch) {
+        final Set<Dpid> sws = new HashSet<Dpid>();
+        final Map<Long, FlowRuleBatchEntry> fmXids = new HashMap<Long, FlowRuleBatchEntry>();
+        OFFlowMod mod = null;
+        for (FlowRuleBatchEntry fbe : batch.getOperations()) {
+            FlowRule flowRule = fbe.getTarget();
+            OpenFlowSwitch sw = controller.getSwitch(Dpid.dpid(flowRule.deviceId().uri()));
+            sws.add(new Dpid(sw.getId()));
+            FlowModBuilder builder = new FlowModBuilder(flowRule, sw.factory());
+            switch (fbe.getOperator()) {
+                case ADD:
+                    mod = builder.buildFlowAdd();
+                    break;
+                case REMOVE:
+                    mod = builder.buildFlowDel();
+                    break;
+                case MODIFY:
+                    mod = builder.buildFlowMod();
+                    break;
+                default:
+                    log.error("Unsupported batch operation {}", fbe.getOperator());
+            }
+            if (mod != null) {
+                sw.sendMsg(mod);
+                fmXids.put(mod.getXid(), fbe);
+            } else {
+                log.error("Conversion of flowrule {} failed.", flowRule);
+            }
+
+        }
+        InstallationFuture installation = new InstallationFuture(sws, fmXids);
+        for (Long xid : fmXids.keySet()) {
+            pendingFMs.put(xid, installation);
+        }
+        pendingFutures.put(U32.f(batch.hashCode()), installation);
+        installation.verify(batch.hashCode());
+        return installation;
+    }
+
 
 
-    //TODO: InternalFlowRuleProvider listening to stats and error and flowremoved.
-    // possibly barriers as well. May not be internal at all...
     private class InternalFlowProvider
     private class InternalFlowProvider
     implements OpenFlowSwitchListener, OpenFlowEventListener {
     implements OpenFlowSwitchListener, OpenFlowEventListener {
 
 
@@ -175,7 +228,6 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
             InstallationFuture future = null;
             InstallationFuture future = null;
             switch (msg.getType()) {
             switch (msg.getType()) {
             case FLOW_REMOVED:
             case FLOW_REMOVED:
-                //TODO: make this better
                 OFFlowRemoved removed = (OFFlowRemoved) msg;
                 OFFlowRemoved removed = (OFFlowRemoved) msg;
 
 
                 FlowEntry fr = new FlowEntryBuilder(dpid, removed).build();
                 FlowEntry fr = new FlowEntryBuilder(dpid, removed).build();
@@ -191,7 +243,7 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
                 }
                 }
                 break;
                 break;
             case ERROR:
             case ERROR:
-                future = pendingFutures.get(msg.getXid());
+                future = pendingFMs.get(msg.getXid());
                 if (future != null) {
                 if (future != null) {
                     future.fail((OFErrorMsg) msg, dpid);
                     future.fail((OFErrorMsg) msg, dpid);
                 }
                 }
@@ -203,10 +255,7 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
         }
         }
 
 
         @Override
         @Override
-        public void roleAssertFailed(Dpid dpid, RoleState role) {
-            // TODO Auto-generated method stub
-
-        }
+        public void roleAssertFailed(Dpid dpid, RoleState role) {}
 
 
         private synchronized void pushFlowMetrics(Dpid dpid, OFStatsReply stats) {
         private synchronized void pushFlowMetrics(Dpid dpid, OFStatsReply stats) {
             if (stats.getStatsType() != OFStatsType.FLOW) {
             if (stats.getStatsType() != OFStatsType.FLOW) {
@@ -230,7 +279,6 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
         }
         }
 
 
         private boolean tableMissRule(Dpid dpid, OFFlowStatsEntry reply) {
         private boolean tableMissRule(Dpid dpid, OFFlowStatsEntry reply) {
-            // TODO NEED TO FIND A BETTER WAY TO AVOID DOING THIS
             if (reply.getVersion().equals(OFVersion.OF_10) ||
             if (reply.getVersion().equals(OFVersion.OF_10) ||
                     reply.getMatch().getMatchFields().iterator().hasNext()) {
                     reply.getMatch().getMatchFields().iterator().hasNext()) {
                 return false;
                 return false;
@@ -251,104 +299,91 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
             }
             }
             return false;
             return false;
         }
         }
-
-    }
-
-
-    @Override
-    public Future<Void> executeBatch(BatchOperation<FlowRuleBatchEntry> batch) {
-        final Set<Dpid> sws = new HashSet<Dpid>();
-
-        for (FlowRuleBatchEntry fbe : batch.getOperations()) {
-            FlowRule flowRule = fbe.getTarget();
-            OpenFlowSwitch sw = controller.getSwitch(Dpid.dpid(flowRule.deviceId().uri()));
-            sws.add(new Dpid(sw.getId()));
-            switch (fbe.getOperator()) {
-                case ADD:
-                  //TODO: Track XID for each flowmod
-                    sw.sendMsg(new FlowModBuilder(flowRule, sw.factory()).buildFlowAdd());
-                    break;
-                case REMOVE:
-                  //TODO: Track XID for each flowmod
-                    sw.sendMsg(new FlowModBuilder(flowRule, sw.factory()).buildFlowDel());
-                    break;
-                case MODIFY:
-                  //TODO: Track XID for each flowmod
-                    sw.sendMsg(new FlowModBuilder(flowRule, sw.factory()).buildFlowMod());
-                    break;
-                default:
-                    log.error("Unsupported batch operation {}", fbe.getOperator());
-            }
-        }
-        InstallationFuture installation = new InstallationFuture(sws);
-        pendingFutures.put(U32.f(batch.hashCode()), installation);
-        installation.verify(batch.hashCode());
-        return installation;
     }
     }
 
 
-    private class InstallationFuture implements Future<Void> {
+    private class InstallationFuture implements Future<CompletedBatchOperation> {
 
 
         private final Set<Dpid> sws;
         private final Set<Dpid> sws;
         private final AtomicBoolean ok = new AtomicBoolean(true);
         private final AtomicBoolean ok = new AtomicBoolean(true);
+        private final Map<Long, FlowRuleBatchEntry> fms;
+
         private final List<FlowEntry> offendingFlowMods = Lists.newLinkedList();
         private final List<FlowEntry> offendingFlowMods = Lists.newLinkedList();
 
 
         private final CountDownLatch countDownLatch;
         private final CountDownLatch countDownLatch;
+        private Integer pendingXid;
+        private BatchState state;
 
 
-        public InstallationFuture(Set<Dpid> sws) {
+        public InstallationFuture(Set<Dpid> sws, Map<Long, FlowRuleBatchEntry> fmXids) {
+            this.state = BatchState.STARTED;
             this.sws = sws;
             this.sws = sws;
+            this.fms = fmXids;
             countDownLatch = new CountDownLatch(sws.size());
             countDownLatch = new CountDownLatch(sws.size());
         }
         }
 
 
         public void fail(OFErrorMsg msg, Dpid dpid) {
         public void fail(OFErrorMsg msg, Dpid dpid) {
             ok.set(false);
             ok.set(false);
-            //TODO add reason to flowentry
+            FlowEntry fe = null;
+            FlowRuleBatchEntry fbe = fms.get(msg.getXid());
+            FlowRule offending = fbe.getTarget();
             //TODO handle specific error msgs
             //TODO handle specific error msgs
-            //offendingFlowMods.add(new FlowEntryBuilder(dpid, msg.));
             switch (msg.getErrType()) {
             switch (msg.getErrType()) {
                 case BAD_ACTION:
                 case BAD_ACTION:
+                    OFBadActionErrorMsg bad = (OFBadActionErrorMsg) msg;
+                    fe = new DefaultFlowEntry(offending, bad.getErrType().ordinal(),
+                            bad.getCode().ordinal());
                     break;
                     break;
                 case BAD_INSTRUCTION:
                 case BAD_INSTRUCTION:
+                    OFBadInstructionErrorMsg badins = (OFBadInstructionErrorMsg) msg;
+                    fe = new DefaultFlowEntry(offending, badins.getErrType().ordinal(),
+                            badins.getCode().ordinal());
                     break;
                     break;
                 case BAD_MATCH:
                 case BAD_MATCH:
+                    OFBadMatchErrorMsg badMatch = (OFBadMatchErrorMsg) msg;
+                    fe = new DefaultFlowEntry(offending, badMatch.getErrType().ordinal(),
+                            badMatch.getCode().ordinal());
                     break;
                     break;
                 case BAD_REQUEST:
                 case BAD_REQUEST:
-                    break;
-                case EXPERIMENTER:
+                    OFBadRequestErrorMsg badReq = (OFBadRequestErrorMsg) msg;
+                    fe = new DefaultFlowEntry(offending, badReq.getErrType().ordinal(),
+                            badReq.getCode().ordinal());
                     break;
                     break;
                 case FLOW_MOD_FAILED:
                 case FLOW_MOD_FAILED:
+                    OFFlowModFailedErrorMsg fmFail = (OFFlowModFailedErrorMsg) msg;
+                    fe = new DefaultFlowEntry(offending, fmFail.getErrType().ordinal(),
+                            fmFail.getCode().ordinal());
                     break;
                     break;
+                case EXPERIMENTER:
                 case GROUP_MOD_FAILED:
                 case GROUP_MOD_FAILED:
-                    break;
                 case HELLO_FAILED:
                 case HELLO_FAILED:
-                    break;
                 case METER_MOD_FAILED:
                 case METER_MOD_FAILED:
-                    break;
                 case PORT_MOD_FAILED:
                 case PORT_MOD_FAILED:
-                    break;
                 case QUEUE_OP_FAILED:
                 case QUEUE_OP_FAILED:
-                    break;
                 case ROLE_REQUEST_FAILED:
                 case ROLE_REQUEST_FAILED:
-                    break;
                 case SWITCH_CONFIG_FAILED:
                 case SWITCH_CONFIG_FAILED:
-                    break;
                 case TABLE_FEATURES_FAILED:
                 case TABLE_FEATURES_FAILED:
-                    break;
                 case TABLE_MOD_FAILED:
                 case TABLE_MOD_FAILED:
+                    fe = new DefaultFlowEntry(offending, msg.getErrType().ordinal(), 0);
                     break;
                     break;
                 default:
                 default:
-                    break;
+                    log.error("Unknown error type {}", msg.getErrType());
 
 
             }
             }
+            offendingFlowMods.add(fe);
 
 
         }
         }
 
 
+
         public void satisfyRequirement(Dpid dpid) {
         public void satisfyRequirement(Dpid dpid) {
             log.warn("Satisfaction from switch {}", dpid);
             log.warn("Satisfaction from switch {}", dpid);
             sws.remove(dpid);
             sws.remove(dpid);
             countDownLatch.countDown();
             countDownLatch.countDown();
+            cleanUp();
+
         }
         }
 
 
+
         public void verify(Integer id) {
         public void verify(Integer id) {
+            pendingXid = id;
             for (Dpid dpid : sws) {
             for (Dpid dpid : sws) {
                 OpenFlowSwitch sw = controller.getSwitch(dpid);
                 OpenFlowSwitch sw = controller.getSwitch(dpid);
                 OFBarrierRequest.Builder builder = sw.factory()
                 OFBarrierRequest.Builder builder = sw.factory()
@@ -356,41 +391,59 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
                         .setXid(id);
                         .setXid(id);
                 sw.sendMsg(builder.build());
                 sw.sendMsg(builder.build());
             }
             }
-
-
         }
         }
 
 
         @Override
         @Override
         public boolean cancel(boolean mayInterruptIfRunning) {
         public boolean cancel(boolean mayInterruptIfRunning) {
-                // TODO Auto-generated method stub
-                return false;
+            this.state = BatchState.CANCELLED;
+            cleanUp();
+            for (FlowRuleBatchEntry fbe : fms.values()) {
+                if (fbe.getOperator() == FlowRuleOperation.ADD ||
+                        fbe.getOperator() == FlowRuleOperation.MODIFY) {
+                    removeFlowRule(fbe.getTarget());
+                } else if (fbe.getOperator() == FlowRuleOperation.REMOVE) {
+                    applyRule(fbe.getTarget());
+                }
+
+            }
+            return isCancelled();
         }
         }
 
 
         @Override
         @Override
         public boolean isCancelled() {
         public boolean isCancelled() {
-            // TODO Auto-generated method stub
-            return false;
+            return this.state == BatchState.CANCELLED;
         }
         }
 
 
         @Override
         @Override
         public boolean isDone() {
         public boolean isDone() {
-            return sws.isEmpty();
+            return this.state == BatchState.FINISHED;
         }
         }
 
 
         @Override
         @Override
-        public Void get() throws InterruptedException, ExecutionException {
+        public CompletedBatchOperation get() throws InterruptedException, ExecutionException {
             countDownLatch.await();
             countDownLatch.await();
-            //return offendingFlowMods;
-            return null;
+            this.state = BatchState.FINISHED;
+            return new CompletedBatchOperation(ok.get(), offendingFlowMods);
         }
         }
 
 
         @Override
         @Override
-        public Void get(long timeout, TimeUnit unit)
+        public CompletedBatchOperation get(long timeout, TimeUnit unit)
                 throws InterruptedException, ExecutionException,
                 throws InterruptedException, ExecutionException,
                 TimeoutException {
                 TimeoutException {
-            countDownLatch.await(timeout, unit);
-            //return offendingFlowMods;
-            return null;
+            if (countDownLatch.await(timeout, unit)) {
+                this.state = BatchState.FINISHED;
+                return new CompletedBatchOperation(ok.get(), offendingFlowMods);
+            }
+            throw new TimeoutException();
+        }
+
+        private void cleanUp() {
+            if (sws.isEmpty()) {
+                pendingFutures.remove(pendingXid);
+                for (Long xid : fms.keySet()) {
+                    pendingFMs.remove(xid);
+                }
+            }
         }
         }
 
 
     }
     }

 package org.onlab.onos.provider.of.host.impl;
 package org.onlab.onos.provider.of.host.impl;
 
 
-import static com.google.common.collect.Sets.newHashSet;
-import static org.onlab.onos.net.DeviceId.deviceId;
-import static org.onlab.onos.net.PortNumber.portNumber;
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.Set;
-
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -36,6 +29,10 @@ import org.onlab.packet.IpPrefix;
 import org.onlab.packet.VlanId;
 import org.onlab.packet.VlanId;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 
 
+import static org.onlab.onos.net.DeviceId.deviceId;
+import static org.onlab.onos.net.PortNumber.portNumber;
+import static org.slf4j.LoggerFactory.getLogger;
+
 /**
 /**
  * Provider which uses an OpenFlow controller to detect network
  * Provider which uses an OpenFlow controller to detect network
  * end-station hosts.
  * end-station hosts.
@@ -58,6 +55,8 @@ public class OpenFlowHostProvider extends AbstractProvider implements HostProvid
 
 
     private final InternalHostProvider listener = new InternalHostProvider();
     private final InternalHostProvider listener = new InternalHostProvider();
 
 
+    private boolean ipLearn = true;
+
     /**
     /**
      * Creates an OpenFlow host provider.
      * Creates an OpenFlow host provider.
      */
      */
@@ -69,7 +68,6 @@ public class OpenFlowHostProvider extends AbstractProvider implements HostProvid
     public void activate() {
     public void activate() {
         providerService = providerRegistry.register(this);
         providerService = providerRegistry.register(this);
         controller.addPacketListener(10, listener);
         controller.addPacketListener(10, listener);
-
         log.info("Started");
         log.info("Started");
     }
     }
 
 
@@ -78,7 +76,6 @@ public class OpenFlowHostProvider extends AbstractProvider implements HostProvid
         providerRegistry.unregister(this);
         providerRegistry.unregister(this);
         controller.removePacketListener(listener);
         controller.removePacketListener(listener);
         providerService = null;
         providerService = null;
-
         log.info("Stopped");
         log.info("Stopped");
     }
     }
 
 
@@ -95,33 +92,33 @@ public class OpenFlowHostProvider extends AbstractProvider implements HostProvid
 
 
             VlanId vlan = VlanId.vlanId(eth.getVlanID());
             VlanId vlan = VlanId.vlanId(eth.getVlanID());
             ConnectPoint heardOn = new ConnectPoint(deviceId(Dpid.uri(pktCtx.dpid())),
             ConnectPoint heardOn = new ConnectPoint(deviceId(Dpid.uri(pktCtx.dpid())),
-                    portNumber(pktCtx.inPort()));
+                                                    portNumber(pktCtx.inPort()));
 
 
-         // If this is not an edge port, bail out.
+            // If this is not an edge port, bail out.
             Topology topology = topologyService.currentTopology();
             Topology topology = topologyService.currentTopology();
             if (topologyService.isInfrastructure(topology, heardOn)) {
             if (topologyService.isInfrastructure(topology, heardOn)) {
                 return;
                 return;
             }
             }
 
 
             HostLocation hloc = new HostLocation(deviceId(Dpid.uri(pktCtx.dpid())),
             HostLocation hloc = new HostLocation(deviceId(Dpid.uri(pktCtx.dpid())),
-                    portNumber(pktCtx.inPort()),
-                    System.currentTimeMillis());
+                                                 portNumber(pktCtx.inPort()),
+                                                 System.currentTimeMillis());
+
             HostId hid = HostId.hostId(eth.getSourceMAC(), vlan);
             HostId hid = HostId.hostId(eth.getSourceMAC(), vlan);
+
             // Potentially a new or moved host
             // Potentially a new or moved host
             if (eth.getEtherType() == Ethernet.TYPE_ARP) {
             if (eth.getEtherType() == Ethernet.TYPE_ARP) {
-
-
                 ARP arp = (ARP) eth.getPayload();
                 ARP arp = (ARP) eth.getPayload();
-                Set<IpPrefix> ips = newHashSet(IpPrefix.valueOf(arp.getSenderProtocolAddress()));
+                IpPrefix ip = IpPrefix.valueOf(arp.getSenderProtocolAddress());
                 HostDescription hdescr =
                 HostDescription hdescr =
-                        new DefaultHostDescription(eth.getSourceMAC(), vlan, hloc, ips);
+                        new DefaultHostDescription(eth.getSourceMAC(), vlan, hloc, ip);
                 providerService.hostDetected(hid, hdescr);
                 providerService.hostDetected(hid, hdescr);
 
 
-            } else if (eth.getEtherType() == Ethernet.TYPE_IPV4) {
-                IPv4 ip = (IPv4) eth.getPayload();
-                Set<IpPrefix> ips = newHashSet(IpPrefix.valueOf(ip.getSourceAddress()));
+            } else if (ipLearn && eth.getEtherType() == Ethernet.TYPE_IPV4) {
+                IPv4 pip = (IPv4) eth.getPayload();
+                IpPrefix ip = IpPrefix.valueOf(pip.getSourceAddress());
                 HostDescription hdescr =
                 HostDescription hdescr =
-                        new DefaultHostDescription(eth.getSourceMAC(), vlan, hloc, ips);
+                        new DefaultHostDescription(eth.getSourceMAC(), vlan, hloc, ip);
                 providerService.hostDetected(hid, hdescr);
                 providerService.hostDetected(hid, hdescr);
 
 
             }
             }

 package org.onlab.onos.provider.of.packet.impl;
 package org.onlab.onos.provider.of.packet.impl;
 
 
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.List;
-
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.flow.instructions.Instruction;
 import org.onlab.onos.net.flow.instructions.Instruction;
 import org.onlab.onos.net.flow.instructions.Instruction.Type;
 import org.onlab.onos.net.flow.instructions.Instruction.Type;
@@ -14,16 +10,16 @@ import org.onlab.onos.net.packet.OutboundPacket;
 import org.onlab.onos.openflow.controller.OpenFlowPacketContext;
 import org.onlab.onos.openflow.controller.OpenFlowPacketContext;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.Ethernet;
 import org.projectfloodlight.openflow.types.OFPort;
 import org.projectfloodlight.openflow.types.OFPort;
-import org.slf4j.Logger;
 
 
-public class OpenFlowCorePacketContext extends DefaultPacketContext {
+import java.util.List;
 
 
-    private final Logger log = getLogger(getClass());
+public class OpenFlowCorePacketContext extends DefaultPacketContext {
 
 
     private final OpenFlowPacketContext ofPktCtx;
     private final OpenFlowPacketContext ofPktCtx;
 
 
     protected OpenFlowCorePacketContext(long time, InboundPacket inPkt,
     protected OpenFlowCorePacketContext(long time, InboundPacket inPkt,
-            OutboundPacket outPkt, boolean block, OpenFlowPacketContext ofPktCtx) {
+                                        OutboundPacket outPkt, boolean block,
+                                        OpenFlowPacketContext ofPktCtx) {
         super(time, inPkt, outPkt, block);
         super(time, inPkt, outPkt, block);
         this.ofPktCtx = ofPktCtx;
         this.ofPktCtx = ofPktCtx;
     }
     }
@@ -36,9 +32,8 @@ public class OpenFlowCorePacketContext extends DefaultPacketContext {
             } else {
             } else {
                 Ethernet eth = new Ethernet();
                 Ethernet eth = new Ethernet();
                 eth.deserialize(outPacket().data().array(), 0,
                 eth.deserialize(outPacket().data().array(), 0,
-                        outPacket().data().array().length);
+                                outPacket().data().array().length);
                 sendPacket(eth);
                 sendPacket(eth);
-
             }
             }
 
 
         }
         }
@@ -61,6 +56,7 @@ public class OpenFlowCorePacketContext extends DefaultPacketContext {
         }
         }
         ofPktCtx.send();
         ofPktCtx.send();
     }
     }
+
     private OFPort buildPort(PortNumber port) {
     private OFPort buildPort(PortNumber port) {
         return OFPort.of((int) port.toLong());
         return OFPort.of((int) port.toLong());
     }
     }

@@ -6,7 +6,7 @@
 export ONOS_ROOT=${ONOS_ROOT:-~/onos-next}
 export ONOS_ROOT=${ONOS_ROOT:-~/onos-next}
 
 
 # Setup some environmental context for developers
 # Setup some environmental context for developers
-export JAVA_HOME=$(/usr/libexec/java_home -v 1.7)
+export JAVA_HOME=${JAVA_HOME:-$(/usr/libexec/java_home -v 1.7)}
 export MAVEN=${MAVEN:-~/Applications/apache-maven-3.2.2}
 export MAVEN=${MAVEN:-~/Applications/apache-maven-3.2.2}
 export KARAF=${KARAF:-~/Applications/apache-karaf-3.0.1}
 export KARAF=${KARAF:-~/Applications/apache-karaf-3.0.1}
 export KARAF_LOG=$KARAF/data/log/karaf.log
 export KARAF_LOG=$KARAF/data/log/karaf.log
@@ -33,6 +33,7 @@ alias obs='onos-build-selective'
 alias op='onos-package'
 alias op='onos-package'
 alias ot='onos-test'
 alias ot='onos-test'
 alias ol='onos-log'
 alias ol='onos-log'
+alias ow='onos-watch'
 alias go='ob && ot && onos -w'
 alias go='ob && ot && onos -w'
 alias pub='onos-push-update-bundle'
 alias pub='onos-push-update-bundle'
 
 

+#!/bin/bash
 #------------------------------------------------------------------------------
 #------------------------------------------------------------------------------
-# Echoes project-level directory if a Java file within is newer than its
-# class file counterpart
+# Echoes project-level directory if a Java file within is newer than the
+# target directory.
 #------------------------------------------------------------------------------
 #------------------------------------------------------------------------------
 
 
 javaFile=${1#*\/src\/*\/java/}
 javaFile=${1#*\/src\/*\/java/}
@@ -10,9 +11,7 @@ basename=${1/*\//}
 
 
 src=${1/$javaFile/}
 src=${1/$javaFile/}
 project=${src/src*/}
 project=${src/src*/}
-classFile=${javaFile/.java/.class}
+target=$project/target
 
 
-[ ${project}target/classes/$classFile -nt ${src}$javaFile -o \
-    ${project}target/test-classes/$classFile -nt ${src}$javaFile ] \
-    || echo ${src/src*/}
+[ $target -nt ${src}$javaFile ] || echo ${src/src*/}
 
 

+#!/bin/bash
+
+kpid=$(ps -ef | grep karaf.main.Main | grep -v grep | cut -c10-15 | tr -d ' ')
+
+[ -z "$kpid" ] && echo "No ONOS!" && exit 1
+
+/opt/jprofiler8/bin/jpenable --gui --port=8849 --pid=$kpid

@@ -7,7 +7,7 @@
 . $ONOS_ROOT/tools/build/envDefaults
 . $ONOS_ROOT/tools/build/envDefaults
 
 
 cd ~/.m2/repository
 cd ~/.m2/repository
-jar=$(find org/onlab -type f -name '*.jar' | grep $1 | grep -v -e -tests | head -n 1)
+jar=$(find org/onlab -type f -name '*.jar' | grep -e $1 | grep -v -e -tests | head -n 1)
 
 
 [ -z "$jar" ] && echo "No bundle $1 found for" && exit 1
 [ -z "$jar" ] && echo "No bundle $1 found for" && exit 1
 
 

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Update bundle on locally running karaf.
+#-------------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+cd ~/.m2/repository
+jar=$(find org/onlab -type f -name '*.jar' | grep -e $1 | grep -v -e -tests | head -n 1)
+
+[ -z "$jar" ] && echo "No bundle $1 found for" && exit 1
+
+bundle=$(echo $(basename $jar .jar) | sed 's/-[0-9].*//g')
+
+client "bundle:update -f $bundle" 2>/dev/null

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Monitors selected set of ONOS commands using the system watch command.
+#-------------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+node=${1:-$OCI}
+
+commands="${2:-summary,intents,flows,hosts}"
+
+aux=/tmp/onos-watch.$$
+trap "rm -f $aux" EXIT
+
+echo "$commands" | tr ',' '\n' > $aux
+watch $3 "onos $node -b <$aux 2>/dev/null"

+# Local VirtualBox-based single ONOS instance & ONOS mininet box
+
+export ONOS_NIC=192.168.56.*
+export OC1="192.168.56.103"
+export OCN="192.168.56.103"
+
+export ONOS_FEATURES="webconsole,onos-api,onos-core-trivial,onos-cli,onos-openflow,onos-app-fwd"

@@ -44,18 +44,10 @@
             <artifactId>minimal-json</artifactId>
             <artifactId>minimal-json</artifactId>
         </dependency>
         </dependency>
         <dependency>
         <dependency>
-          <groupId>com.esotericsoftware.kryo</groupId>
+          <groupId>com.esotericsoftware</groupId>
           <artifactId>kryo</artifactId>
           <artifactId>kryo</artifactId>
         </dependency>
         </dependency>
         <dependency>
         <dependency>
-            <groupId>com.esotericsoftware</groupId>
-            <artifactId>minlog</artifactId>
-        </dependency>
-        <dependency>
-            <groupId>org.objenesis</groupId>
-            <artifactId>objenesis</artifactId>
-        </dependency>
-        <dependency>
             <groupId>io.dropwizard.metrics</groupId>
             <groupId>io.dropwizard.metrics</groupId>
             <artifactId>metrics-core</artifactId>
             <artifactId>metrics-core</artifactId>
             <version>3.1.0</version>
             <version>3.1.0</version>

@@ -22,11 +22,12 @@ import java.util.Arrays;
  *
  *
  */
  */
 public class MacAddress {
 public class MacAddress {
-    public static final byte[] ZERO_MAC_ADDRESS =
-            MacAddress.valueOf("00:00:00:00:00:00").getAddress();
 
 
-    public static final byte[] BROADCAST_MAC =
-            MacAddress.valueOf("ff:ff:ff:ff:ff:ff").getAddress();
+    public static final MacAddress ZERO = valueOf("00:00:00:00:00:00");
+    public static final MacAddress BROADCAST = valueOf("ff:ff:ff:ff:ff:ff");
+
+    public static final byte[] ZERO_MAC_ADDRESS = ZERO.getAddress();
+    public static final byte[] BROADCAST_MAC = BROADCAST.getAddress();
 
 
     public static final int MAC_ADDRESS_LENGTH = 6;
     public static final int MAC_ADDRESS_LENGTH = 6;
     private byte[] address = new byte[MacAddress.MAC_ADDRESS_LENGTH];
     private byte[] address = new byte[MacAddress.MAC_ADDRESS_LENGTH];

@@ -3,12 +3,15 @@ package org.onlab.packet;
 /**
 /**
  * Representation of a VLAN ID.
  * Representation of a VLAN ID.
  */
  */
-// FIXME: This will end-up looking like a constant; we should name it 'VlanId', 'IpAddress', 'MacAddress'.
 public class VlanId {
 public class VlanId {
 
 
     private final short value;
     private final short value;
+
     // Based on convention used elsewhere? Check and change if needed
     // Based on convention used elsewhere? Check and change if needed
     public static final short UNTAGGED = (short) 0xffff;
     public static final short UNTAGGED = (short) 0xffff;
+
+    public static final VlanId NONE = VlanId.vlanId(UNTAGGED);
+
     // A VLAN ID is actually 12 bits of a VLAN tag.
     // A VLAN ID is actually 12 bits of a VLAN tag.
     public static final short MAX_VLAN = 4095;
     public static final short MAX_VLAN = 4095;
 
 

@@ -32,10 +32,6 @@
             <scope>test</scope>
             <scope>test</scope>
         </dependency>
         </dependency>
         <dependency>
         <dependency>
-            <groupId>de.javakaffee</groupId>
-            <artifactId>kryo-serializers</artifactId>
-        </dependency>
-        <dependency>
             <groupId>io.netty</groupId>
             <groupId>io.netty</groupId>
             <artifactId>netty-all</artifactId>
             <artifactId>netty-all</artifactId>
         </dependency>
         </dependency>