Merge branch 'master' of ssh://gerrit.onlab.us:29418/onos-next

Conflicts:
	core/net/src/main/java/org/onlab/onos/net/device/impl/DeviceManager.java
	core/store/hz/cluster/src/main/java/org/onlab/onos/store/cluster/impl/DistributedMastershipStore.java

Change-Id: Ia1274657b27e01366a4a87196a13068d7104ee80

@@ -24,10 +24,20 @@
         </dependency>
         <dependency>
             <groupId>org.onlab.onos</groupId>
+            <artifactId>onlab-osgi</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>org.onlab.onos</groupId>
             <artifactId>onlab-nio</artifactId>
             <version>${project.version}</version>
         </dependency>
         <dependency>
+            <groupId>org.onlab.onos</groupId>
+            <artifactId>onlab-netty</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
             <groupId>org.apache.karaf.shell</groupId>
             <artifactId>org.apache.karaf.shell.console</artifactId>
         </dependency>

@@ -11,6 +11,9 @@ import org.onlab.onos.cluster.ClusterService;
 import org.onlab.onos.net.device.DeviceEvent;
 import org.onlab.onos.net.device.DeviceListener;
 import org.onlab.onos.net.device.DeviceService;
+import org.onlab.onos.net.intent.IntentEvent;
+import org.onlab.onos.net.intent.IntentListener;
+import org.onlab.onos.net.intent.IntentService;
 import org.slf4j.Logger;
 
 import static org.slf4j.LoggerFactory.getLogger;
@@ -29,13 +32,18 @@ public class FooComponent {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected DeviceService deviceService;
 
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected IntentService intentService;
+
     private final ClusterEventListener clusterListener = new InnerClusterListener();
     private final DeviceListener deviceListener = new InnerDeviceListener();
+    private final IntentListener intentListener = new InnerIntentListener();
 
     @Activate
     public void activate() {
         clusterService.addListener(clusterListener);
         deviceService.addListener(deviceListener);
+        intentService.addListener(intentListener);
         log.info("Started");
     }
 
@@ -43,6 +51,7 @@ public class FooComponent {
     public void deactivate() {
         clusterService.removeListener(clusterListener);
         deviceService.removeListener(deviceListener);
+        intentService.removeListener(intentListener);
         log.info("Stopped");
     }
 
@@ -59,6 +68,23 @@ public class FooComponent {
             log.info("YEEEEHAAAAW! {}", event);
         }
     }
+
+    private class InnerIntentListener implements IntentListener {
+        @Override
+        public void event(IntentEvent event) {
+            String message;
+            if (event.type() == IntentEvent.Type.SUBMITTED) {
+                message = "WOW! It looks like someone has some intentions: {}";
+            } else if (event.type() == IntentEvent.Type.INSTALLED) {
+                message = "AWESOME! So far things are going great: {}";
+            } else if (event.type() == IntentEvent.Type.WITHDRAWN) {
+                message = "HMMM! Ambitions are fading apparently: {}";
+            } else {
+                message = "CRAP!!! Things are not turning out as intended: {}";
+            }
+            log.info(message, event.subject());
+        }
+    }
 }
 
 

+package org.onlab.onos.foo;
+
+import java.io.IOException;
+
+import org.onlab.netty.Message;
+import org.onlab.netty.MessageHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+/**
+ * Message handler that echos the message back to the sender.
+ */
+public class NettyEchoHandler implements MessageHandler {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
+    @Override
+    public void handle(Message message) throws IOException {
+        //log.info("Received message. Echoing it back to the sender.");
+        message.respond(message.payload());
+    }
+}

+package org.onlab.onos.foo;
+
+import org.onlab.netty.Message;
+import org.onlab.netty.MessageHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A MessageHandler that simply logs the information.
+ */
+public class NettyLoggingHandler implements MessageHandler {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
+    @Override
+    public void handle(Message message) {
+        //log.info("Received message. Payload has {} bytes", message.payload().length);
+    }
+}

+package org.onlab.onos.foo;
+
+import java.io.IOException;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
+import org.onlab.metrics.MetricsComponent;
+import org.onlab.metrics.MetricsFeature;
+import org.onlab.metrics.MetricsManager;
+import org.onlab.netty.Endpoint;
+import org.onlab.netty.NettyMessagingService;
+import org.onlab.netty.Response;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.codahale.metrics.Timer;
+
+// FIXME: Should be move out to test or app
+public final class SimpleNettyClient {
+
+private static Logger log = LoggerFactory.getLogger(SimpleNettyClient.class);
+
+        private SimpleNettyClient() {
+    }
+
+    public static void main(String[] args)
+            throws IOException, InterruptedException, ExecutionException,
+            TimeoutException {
+        try {
+            startStandalone(args);
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        System.exit(0);
+    }
+    public static void startStandalone(String... args) throws Exception {
+        String host = args.length > 0 ? args[0] : "localhost";
+        int port = args.length > 1 ? Integer.parseInt(args[1]) : 8081;
+        int warmup = args.length > 2 ? Integer.parseInt(args[2]) : 1000;
+        int iterations = args.length > 3 ? Integer.parseInt(args[3]) : 50 * 100000;
+        NettyMessagingService messaging = new TestNettyMessagingService(9081);
+        MetricsManager metrics = new MetricsManager();
+        messaging.activate();
+        metrics.activate();
+        MetricsFeature feature = new MetricsFeature("latency");
+        MetricsComponent component = metrics.registerComponent("NettyMessaging");
+        log.info("warmup....");
+
+        for (int i = 0; i < warmup; i++) {
+            messaging.sendAsync(new Endpoint(host, port), "simple", "Hello World".getBytes());
+            Response response = messaging
+                    .sendAndReceive(new Endpoint(host, port), "echo",
+                            "Hello World".getBytes());
+        }
+
+        log.info("measuring async sender");
+        Timer sendAsyncTimer = metrics.createTimer(component, feature, "AsyncSender");
+
+        for (int i = 0; i < iterations; i++) {
+            Timer.Context context = sendAsyncTimer.time();
+            messaging.sendAsync(new Endpoint(host, port), "simple", "Hello World".getBytes());
+            context.stop();
+        }
+
+        Timer sendAndReceiveTimer = metrics.createTimer(component, feature, "SendAndReceive");
+        for (int i = 0; i < iterations; i++) {
+            Timer.Context context = sendAndReceiveTimer.time();
+            Response response = messaging
+                    .sendAndReceive(new Endpoint(host, port), "echo",
+                                    "Hello World".getBytes());
+            // System.out.println("Got back:" + new String(response.get(2, TimeUnit.SECONDS)));
+            context.stop();
+        }
+    }
+
+    public static class TestNettyMessagingService extends NettyMessagingService {
+        public TestNettyMessagingService(int port) throws Exception {
+            super(port);
+        }
+    }
+}

+package org.onlab.onos.foo;
+
+import static org.onlab.onos.foo.SimpleNettyClient.startStandalone;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cli.AbstractShellCommand;
+
+/**
+ * Test Netty client performance.
+ */
+@Command(scope = "onos", name = "simple-netty-client",
+        description = "Starts the simple Netty client")
+public class SimpleNettyClientCommand extends AbstractShellCommand {
+
+    //FIXME: replace these arguments with proper ones needed for the test.
+    @Argument(index = 0, name = "hostname", description = "Server Hostname",
+            required = false, multiValued = false)
+    String host = "localhost";
+
+    @Argument(index = 3, name = "port", description = "Port",
+            required = false, multiValued = false)
+    String port = "8081";
+
+    @Argument(index = 1, name = "warmupCount", description = "Warm-up count",
+            required = false, multiValued = false)
+    String warmup = "1000";
+
+    @Argument(index = 2, name = "messageCount", description = "Message count",
+            required = false, multiValued = false)
+    String messageCount = "100000";
+
+    @Override
+    protected void execute() {
+        try {
+            startStandalone(new String[]{host, port, warmup, messageCount});
+        } catch (Exception e) {
+            error("Unable to start client %s", e);
+        }
+    }
+
+}

+package org.onlab.onos.foo;
+
+import org.onlab.netty.NettyMessagingService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Test to measure Messaging performance.
+ */
+    public final class SimpleNettyServer {
+        private static Logger log = LoggerFactory.getLogger(SimpleNettyServer.class);
+
+            private SimpleNettyServer() {}
+
+            public static void main(String... args) throws Exception {
+                startStandalone(args);
+                System.exit(0);
+            }
+
+        public static void startStandalone(String[] args) throws Exception {
+            NettyMessagingService server = new NettyMessagingService(8081);
+            server.activate();
+            server.registerHandler("simple", new NettyLoggingHandler());
+            server.registerHandler("echo", new NettyEchoHandler());
+        }
+    }
+

+package org.onlab.onos.foo;
+
+import static org.onlab.onos.foo.SimpleNettyServer.startStandalone;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cli.AbstractShellCommand;
+
+/**
+ * Starts the Simple Netty server.
+ */
+@Command(scope = "onos", name = "simple-netty-server",
+         description = "Starts the simple netty server")
+public class SimpleNettyServerCommand extends AbstractShellCommand {
+
+    //FIXME: Replace these with parameters for
+    @Argument(index = 0, name = "serverIp", description = "Server IP address",
+              required = false, multiValued = false)
+    String serverIp = "127.0.0.1";
+
+    @Argument(index = 1, name = "workers", description = "IO workers",
+              required = false, multiValued = false)
+    String workers = "6";
+
+    @Argument(index = 2, name = "messageLength", description = "Message length (bytes)",
+              required = false, multiValued = false)
+    String messageLength = "128";
+
+    @Override
+    protected void execute() {
+        try {
+            startStandalone(new String[]{serverIp, workers, messageLength});
+        } catch (Exception e) {
+            error("Unable to start server %s", e);
+        }
+    }
+
+}

@@ -7,6 +7,12 @@
         <command>
             <action class="org.onlab.onos.foo.TestIOServerCommand"/>
         </command>
+        <command>
+            <action class="org.onlab.onos.foo.SimpleNettyServerCommand"/>
+        </command>
+        <command>
+            <action class="org.onlab.onos.foo.SimpleNettyClientCommand"/>
+        </command>
     </command-bundle>
 
 </blueprint>

@@ -26,9 +26,7 @@ import org.onlab.onos.net.packet.InboundPacket;
 import org.onlab.onos.net.packet.PacketContext;
 import org.onlab.onos.net.packet.PacketProcessor;
 import org.onlab.onos.net.packet.PacketService;
-import org.onlab.onos.net.proxyarp.ProxyArpService;
 import org.onlab.onos.net.topology.TopologyService;
-import org.onlab.packet.ARP;
 import org.onlab.packet.Ethernet;
 import org.slf4j.Logger;
 
@@ -39,6 +37,7 @@ import org.slf4j.Logger;
 public class ReactiveForwarding {
 
     private static final int TIMEOUT = 10;
+    private static final int PRIORITY = 10;
 
     private final Logger log = getLogger(getClass());
 
@@ -54,9 +53,6 @@ public class ReactiveForwarding {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected FlowRuleService flowRuleService;
 
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected ProxyArpService proxyArpService;
-
     private ReactivePacketProcessor processor = new ReactivePacketProcessor();
 
     private ApplicationId appId;
@@ -64,7 +60,7 @@ public class ReactiveForwarding {
     @Activate
     public void activate() {
         appId = ApplicationId.getAppId();
-        packetService.addProcessor(processor, PacketProcessor.ADVISOR_MAX + 1);
+        packetService.addProcessor(processor, PacketProcessor.ADVISOR_MAX + 2);
         log.info("Started with Application ID {}", appId.id());
     }
 
@@ -92,16 +88,6 @@ public class ReactiveForwarding {
 
             InboundPacket pkt = context.inPacket();
             Ethernet ethPkt = pkt.parsed();
-            if (ethPkt.getEtherType() == Ethernet.TYPE_ARP) {
-                ARP arp = (ARP) ethPkt.getPayload();
-                if (arp.getOpCode() == ARP.OP_REPLY) {
-                    proxyArpService.forward(ethPkt);
-                } else if (arp.getOpCode() == ARP.OP_REQUEST) {
-                    proxyArpService.reply(ethPkt);
-                }
-                context.block();
-                return;
-            }
 
             HostId id = HostId.hostId(ethPkt.getDestinationMAC());
 
@@ -180,24 +166,24 @@ public class ReactiveForwarding {
         // We don't yet support bufferids in the flowservice so packet out first.
         packetOut(context, portNumber);
 
-        if (context.inPacket().parsed().getEtherType() == Ethernet.TYPE_IPV4) {
 
-            // Install the flow rule to handle this type of message from now on.
-            Ethernet inPkt = context.inPacket().parsed();
-            TrafficSelector.Builder builder = new DefaultTrafficSelector.Builder();
-            builder.matchEthType(inPkt.getEtherType())
-                .matchEthSrc(inPkt.getSourceMAC())
-                .matchEthDst(inPkt.getDestinationMAC())
-                .matchInport(context.inPacket().receivedFrom().port());
 
-            TrafficTreatment.Builder treat = new DefaultTrafficTreatment.Builder();
-            treat.setOutput(portNumber);
+        // Install the flow rule to handle this type of message from now on.
+        Ethernet inPkt = context.inPacket().parsed();
+        TrafficSelector.Builder builder = DefaultTrafficSelector.builder();
+        builder.matchEthType(inPkt.getEtherType())
+        .matchEthSrc(inPkt.getSourceMAC())
+        .matchEthDst(inPkt.getDestinationMAC())
+        .matchInport(context.inPacket().receivedFrom().port());
 
-            FlowRule f = new DefaultFlowRule(context.inPacket().receivedFrom().deviceId(),
-                    builder.build(), treat.build(), 0, appId, TIMEOUT);
+        TrafficTreatment.Builder treat = DefaultTrafficTreatment.builder();
+        treat.setOutput(portNumber);
+
+        FlowRule f = new DefaultFlowRule(context.inPacket().receivedFrom().deviceId(),
+                builder.build(), treat.build(), PRIORITY, appId, TIMEOUT);
+
+        flowRuleService.applyFlowRules(f);
 
-            flowRuleService.applyFlowRules(f);
-        }
     }
 
 }

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.onlab.onos</groupId>
+        <artifactId>onos-apps</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>onos-app-ifwd</artifactId>
+    <packaging>bundle</packaging>
+
+    <description>ONOS simple reactive forwarding app that uses intent service</description>
+
+</project>

+package org.onlab.onos.ifwd;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.onlab.onos.net.Host;
+import org.onlab.onos.net.HostId;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.flow.DefaultTrafficSelector;
+import org.onlab.onos.net.flow.DefaultTrafficTreatment;
+import org.onlab.onos.net.flow.TrafficSelector;
+import org.onlab.onos.net.flow.TrafficTreatment;
+import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.intent.HostToHostIntent;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.IntentService;
+import org.onlab.onos.net.packet.DefaultOutboundPacket;
+import org.onlab.onos.net.packet.InboundPacket;
+import org.onlab.onos.net.packet.OutboundPacket;
+import org.onlab.onos.net.packet.PacketContext;
+import org.onlab.onos.net.packet.PacketProcessor;
+import org.onlab.onos.net.packet.PacketService;
+import org.onlab.onos.net.topology.TopologyService;
+import org.onlab.packet.Ethernet;
+import org.slf4j.Logger;
+
+/**
+ * WORK-IN-PROGRESS: Sample reactive forwarding application using intent framework.
+ */
+@Component(immediate = true)
+public class IntentReactiveForwarding {
+
+    private final Logger log = getLogger(getClass());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected TopologyService topologyService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected PacketService packetService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected IntentService intentService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostService hostService;
+
+    private ReactivePacketProcessor processor = new ReactivePacketProcessor();
+
+    private static long intentId = 0x123000;
+
+    @Activate
+    public void activate() {
+        packetService.addProcessor(processor, PacketProcessor.ADVISOR_MAX + 2);
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        packetService.removeProcessor(processor);
+        processor = null;
+        log.info("Stopped");
+    }
+
+    /**
+     * Packet processor responsible for forwarding packets along their paths.
+     */
+    private class ReactivePacketProcessor implements PacketProcessor {
+
+        @Override
+        public void process(PacketContext context) {
+            // Stop processing if the packet has been handled, since we
+            // can't do any more to it.
+            if (context.isHandled()) {
+                return;
+            }
+
+            InboundPacket pkt = context.inPacket();
+            Ethernet ethPkt = pkt.parsed();
+
+            HostId srcId = HostId.hostId(ethPkt.getSourceMAC());
+            HostId dstId = HostId.hostId(ethPkt.getDestinationMAC());
+
+            // Do we know who this is for? If not, flood and bail.
+            Host dst = hostService.getHost(dstId);
+            if (dst == null) {
+                flood(context);
+                return;
+            }
+
+            // Otherwise forward and be done with it.
+            setUpConnectivity(context, srcId, dstId);
+            forwardPacketToDst(context, dst);
+        }
+    }
+
+    // Floods the specified packet if permissible.
+    private void flood(PacketContext context) {
+        if (topologyService.isBroadcastPoint(topologyService.currentTopology(),
+                                             context.inPacket().receivedFrom())) {
+            packetOut(context, PortNumber.FLOOD);
+        } else {
+            context.block();
+        }
+    }
+
+    // Sends a packet out the specified port.
+    private void packetOut(PacketContext context, PortNumber portNumber) {
+        context.treatmentBuilder().setOutput(portNumber);
+        context.send();
+    }
+
+    private void forwardPacketToDst(PacketContext context, Host dst) {
+        TrafficTreatment treatment = DefaultTrafficTreatment.builder().setOutput(dst.location().port()).build();
+        OutboundPacket packet = new DefaultOutboundPacket(dst.location().deviceId(),
+                                                          treatment, context.inPacket().unparsed());
+        packetService.emit(packet);
+        log.info("sending packet: {}", packet);
+    }
+
+    // Install a rule forwarding the packet to the specified port.
+    private void setUpConnectivity(PacketContext context, HostId srcId, HostId dstId) {
+        TrafficSelector selector = DefaultTrafficSelector.builder().build();
+        TrafficTreatment treatment = DefaultTrafficTreatment.builder().build();
+
+        HostToHostIntent intent =
+                new HostToHostIntent(new IntentId(intentId++), srcId, dstId,
+                                     selector, treatment);
+
+        intentService.submit(intent);
+    }
+
+}

+/**
+ * Trivial application that provides simple form of reactive forwarding
+ * using the intent service.
+ */
+package org.onlab.onos.ifwd;

@@ -19,8 +19,10 @@
     <modules>
         <module>tvue</module>
         <module>fwd</module>
+        <module>ifwd</module>
         <module>foo</module>
         <module>mobility</module>
+        <module>proxyarp</module>
 	    <module>config</module>
     </modules>
 

+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>org.onlab.onos</groupId>
+        <artifactId>onos-apps</artifactId>
+        <version>1.0.0-SNAPSHOT</version>
+        <relativePath>../pom.xml</relativePath>
+    </parent>
+
+    <artifactId>onos-app-proxyarp</artifactId>
+    <packaging>bundle</packaging>
+
+    <description>ONOS simple proxy arp module</description>
+
+</project>

+package org.onlab.onos.proxyarp;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.onlab.onos.ApplicationId;
+import org.onlab.onos.net.packet.PacketContext;
+import org.onlab.onos.net.packet.PacketProcessor;
+import org.onlab.onos.net.packet.PacketService;
+import org.onlab.onos.net.proxyarp.ProxyArpService;
+import org.slf4j.Logger;
+
+/**
+ * Sample reactive proxy arp application.
+ */
+@Component(immediate = true)
+public class ProxyArp {
+
+
+    private final Logger log = getLogger(getClass());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected PacketService packetService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ProxyArpService proxyArpService;
+
+    private ProxyArpProcessor processor = new ProxyArpProcessor();
+
+    private ApplicationId appId;
+
+    @Activate
+    public void activate() {
+        appId = ApplicationId.getAppId();
+        packetService.addProcessor(processor, PacketProcessor.ADVISOR_MAX + 1);
+        log.info("Started with Application ID {}", appId.id());
+    }
+
+    @Deactivate
+    public void deactivate() {
+        packetService.removeProcessor(processor);
+        processor = null;
+        log.info("Stopped");
+    }
+
+
+    /**
+     * Packet processor responsible for forwarding packets along their paths.
+     */
+    private class ProxyArpProcessor implements PacketProcessor {
+
+        @Override
+        public void process(PacketContext context) {
+            // Stop processing if the packet has been handled, since we
+            // can't do any more to it.
+            if (context.isHandled()) {
+                return;
+            }
+
+            //handle the arp packet.
+            proxyArpService.handleArp(context);
+        }
+    }
+}
+
+

+/**
+ * Proxy Arp  application that handles arp resolution for you.
+ */
+package org.onlab.onos.proxyarp;

+package org.onlab.onos.cli;
+
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.CoreService;
+import org.onlab.onos.cluster.ClusterService;
+import org.onlab.onos.net.device.DeviceService;
+import org.onlab.onos.net.flow.FlowRuleService;
+import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.intent.IntentService;
+import org.onlab.onos.net.link.LinkService;
+import org.onlab.onos.net.topology.Topology;
+import org.onlab.onos.net.topology.TopologyService;
+
+/**
+ * Provides summary of ONOS model.
+ */
+@Command(scope = "onos", name = "summary",
+         description = "Provides summary of ONOS model")
+public class SummaryCommand extends AbstractShellCommand {
+
+    @Override
+    protected void execute() {
+        TopologyService topologyService = get(TopologyService.class);
+        Topology topology = topologyService.currentTopology();
+        print("version=%s, nodes=%d, devices=%d, links=%d, hosts=%d, clusters=%s, paths=%d, flows=%d, intents=%d",
+              get(CoreService.class).version().toString(),
+              get(ClusterService.class).getNodes().size(),
+              get(DeviceService.class).getDeviceCount(),
+              get(LinkService.class).getLinkCount(),
+              get(HostService.class).getHostCount(),
+              topologyService.getClusters(topology).size(),
+              topology.pathCount(),
+              get(FlowRuleService.class).getFlowRuleCount(),
+              get(IntentService.class).getIntentCount());
+    }
+
+}

+package org.onlab.onos.cli.net;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cli.AbstractShellCommand;
+import org.onlab.onos.net.HostId;
+import org.onlab.onos.net.flow.DefaultTrafficSelector;
+import org.onlab.onos.net.flow.DefaultTrafficTreatment;
+import org.onlab.onos.net.flow.TrafficSelector;
+import org.onlab.onos.net.flow.TrafficTreatment;
+import org.onlab.onos.net.intent.HostToHostIntent;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.IntentService;
+
+/**
+ * Installs host-to-host connectivity intent.
+ */
+@Command(scope = "onos", name = "add-host-intent",
+         description = "Installs host-to-host connectivity intent")
+public class AddHostToHostIntentCommand extends AbstractShellCommand {
+
+    @Argument(index = 0, name = "one", description = "One host ID",
+              required = true, multiValued = false)
+    String one = null;
+
+    @Argument(index = 1, name = "two", description = "Another host ID",
+              required = true, multiValued = false)
+    String two = null;
+
+    private static long id = 0x7870001;
+
+    @Override
+    protected void execute() {
+        IntentService service = get(IntentService.class);
+
+        HostId oneId = HostId.hostId(one);
+        HostId twoId = HostId.hostId(two);
+
+        TrafficSelector selector = DefaultTrafficSelector.builder().build();
+        TrafficTreatment treatment = DefaultTrafficTreatment.builder().build();
+
+        HostToHostIntent intent =
+                new HostToHostIntent(new IntentId(id++), oneId, twoId,
+                                     selector, treatment);
+        service.submit(intent);
+    }
+
+}

+package org.onlab.onos.cli.net;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cli.AbstractShellCommand;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.flow.DefaultTrafficSelector;
+import org.onlab.onos.net.flow.DefaultTrafficTreatment;
+import org.onlab.onos.net.flow.TrafficSelector;
+import org.onlab.onos.net.flow.TrafficTreatment;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.IntentService;
+import org.onlab.onos.net.intent.PointToPointIntent;
+import org.onlab.packet.Ethernet;
+
+/**
+ * Installs point-to-point connectivity intents.
+ */
+@Command(scope = "onos", name = "add-point-intent",
+         description = "Installs point-to-point connectivity intent")
+public class AddPointToPointIntentCommand extends AbstractShellCommand {
+
+    @Argument(index = 0, name = "ingressDevice",
+              description = "Ingress Device/Port Description",
+              required = true, multiValued = false)
+    String ingressDeviceString = null;
+
+    @Argument(index = 1, name = "egressDevice",
+              description = "Egress Device/Port Description",
+              required = true, multiValued = false)
+    String egressDeviceString = null;
+
+    private static long id = 0x7470001;
+
+    @Override
+    protected void execute() {
+        IntentService service = get(IntentService.class);
+
+        DeviceId ingressDeviceId = DeviceId.deviceId(getDeviceId(ingressDeviceString));
+        PortNumber ingressPortNumber =
+                PortNumber.portNumber(getPortNumber(ingressDeviceString));
+        ConnectPoint ingress = new ConnectPoint(ingressDeviceId, ingressPortNumber);
+
+        DeviceId egressDeviceId = DeviceId.deviceId(getDeviceId(egressDeviceString));
+        PortNumber egressPortNumber =
+                PortNumber.portNumber(getPortNumber(egressDeviceString));
+        ConnectPoint egress = new ConnectPoint(egressDeviceId, egressPortNumber);
+
+        TrafficSelector selector = DefaultTrafficSelector.builder()
+                .matchEthType(Ethernet.TYPE_IPV4)
+                .build();
+        TrafficTreatment treatment = DefaultTrafficTreatment.builder().build();
+
+        Intent intent =
+                new PointToPointIntent(new IntentId(id++),
+                                                 selector,
+                                                 treatment,
+                                                 ingress,
+                                                 egress);
+        service.submit(intent);
+    }
+
+    /**
+     * Extracts the port number portion of the ConnectPoint.
+     *
+     * @param deviceString string representing the device/port
+     * @return port number as a string, empty string if the port is not found
+     */
+    private String getPortNumber(String deviceString) {
+        int slash = deviceString.indexOf('/');
+        if (slash <= 0) {
+            return "";
+        }
+        return deviceString.substring(slash + 1, deviceString.length());
+    }
+
+    /**
+     * Extracts the device ID portion of the ConnectPoint.
+     *
+     * @param deviceString string representing the device/port
+     * @return device ID string
+     */
+    private String getDeviceId(String deviceString) {
+        int slash = deviceString.indexOf('/');
+        if (slash <= 0) {
+            return "";
+        }
+        return deviceString.substring(0, slash);
+    }
+}

+package org.onlab.onos.cli.net;
+
+import java.util.List;
+import java.util.SortedSet;
+
+import org.apache.karaf.shell.console.Completer;
+import org.apache.karaf.shell.console.completer.StringsCompleter;
+import org.onlab.onos.cli.AbstractShellCommand;
+import org.onlab.onos.net.Device;
+import org.onlab.onos.net.Port;
+import org.onlab.onos.net.device.DeviceService;
+
+/**
+ * ConnectPoint completer.
+ */
+public class ConnectPointCompleter implements Completer {
+    @Override
+    public int complete(String buffer, int cursor, List<String> candidates) {
+        // Delegate string completer
+        StringsCompleter delegate = new StringsCompleter();
+
+        // Fetch our service and feed it's offerings to the string completer
+        DeviceService service = AbstractShellCommand.get(DeviceService.class);
+
+        // Generate the device ID/port number identifiers
+        for (Device device : service.getDevices()) {
+            SortedSet<String> strings = delegate.getStrings();
+
+            for (Port port : service.getPorts(device.id())) {
+                strings.add(device.id().toString() + "/" + port.number());
+            }
+        }
+
+        // Now let the completer do the work for figuring out what to offer.
+        return delegate.complete(buffer, cursor, candidates);
+    }
+
+}

@@ -35,7 +35,7 @@ public class DevicesListCommand extends AbstractShellCommand {
      * @param service device service
      * @return sorted device list
      */
-    protected List<Device> getSortedDevices(DeviceService service) {
+    protected static List<Device> getSortedDevices(DeviceService service) {
         List<Device> devices = newArrayList(service.getDevices());
         Collections.sort(devices, Comparators.ELEMENT_COMPARATOR);
         return devices;

@@ -5,7 +5,7 @@ import java.util.SortedSet;
 
 import org.apache.karaf.shell.console.Completer;
 import org.apache.karaf.shell.console.completer.StringsCompleter;
-import org.onlab.onos.net.flow.FlowRule.FlowRuleState;
+import org.onlab.onos.net.flow.FlowEntry.FlowEntryState;
 
 /**
  * Device ID completer.
@@ -16,7 +16,7 @@ public class FlowRuleStatusCompleter implements Completer {
         // Delegate string completer
         StringsCompleter delegate = new StringsCompleter();
 
-        FlowRuleState[] states = FlowRuleState.values();
+        FlowEntryState[] states = FlowEntryState.values();
         SortedSet<String> strings = delegate.getStrings();
         for (int i = 0; i < states.length; i++) {
             strings.add(states[i].toString().toLowerCase());

 package org.onlab.onos.cli.net;
 
 import static com.google.common.collect.Lists.newArrayList;
+import static org.onlab.onos.cli.net.DevicesListCommand.getSortedDevices;
 
 import java.util.Collections;
 import java.util.List;
@@ -13,8 +14,8 @@ import org.onlab.onos.cli.Comparators;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.device.DeviceService;
-import org.onlab.onos.net.flow.FlowRule;
-import org.onlab.onos.net.flow.FlowRule.FlowRuleState;
+import org.onlab.onos.net.flow.FlowEntry;
+import org.onlab.onos.net.flow.FlowEntry.FlowEntryState;
 import org.onlab.onos.net.flow.FlowRuleService;
 
 import com.google.common.collect.Maps;
@@ -45,8 +46,8 @@ public class FlowsListCommand extends AbstractShellCommand {
     protected void execute() {
         DeviceService deviceService = get(DeviceService.class);
         FlowRuleService service = get(FlowRuleService.class);
-        Map<Device, List<FlowRule>> flows = getSortedFlows(deviceService, service);
-        for (Device d : flows.keySet()) {
+        Map<Device, List<FlowEntry>> flows = getSortedFlows(deviceService, service);
+        for (Device d : getSortedDevices(deviceService)) {
             printFlows(d, flows.get(d));
         }
     }
@@ -57,21 +58,22 @@ public class FlowsListCommand extends AbstractShellCommand {
      * @param service device service
      * @return sorted device list
      */
-    protected Map<Device, List<FlowRule>> getSortedFlows(DeviceService deviceService, FlowRuleService service) {
-        Map<Device, List<FlowRule>> flows = Maps.newHashMap();
-        List<FlowRule> rules;
-        FlowRuleState s = null;
+    protected Map<Device, List<FlowEntry>> getSortedFlows(DeviceService deviceService,
+                                                          FlowRuleService service) {
+        Map<Device, List<FlowEntry>> flows = Maps.newHashMap();
+        List<FlowEntry> rules;
+        FlowEntryState s = null;
         if (state != null && !state.equals("any")) {
-            s = FlowRuleState.valueOf(state.toUpperCase());
+            s = FlowEntryState.valueOf(state.toUpperCase());
         }
-        Iterable<Device> devices = uri == null ?  deviceService.getDevices() :
+        Iterable<Device> devices = uri == null ? deviceService.getDevices() :
             Collections.singletonList(deviceService.getDevice(DeviceId.deviceId(uri)));
         for (Device d : devices) {
             if (s == null) {
                 rules = newArrayList(service.getFlowEntries(d.id()));
             } else {
                 rules = newArrayList();
-                for (FlowRule f : service.getFlowEntries(d.id())) {
+                for (FlowEntry f : service.getFlowEntries(d.id())) {
                     if (f.state().equals(s)) {
                         rules.add(f);
                     }
@@ -88,19 +90,17 @@ public class FlowsListCommand extends AbstractShellCommand {
      * @param d the device
      * @param flows the set of flows for that device.
      */
-    protected void printFlows(Device d, List<FlowRule> flows) {
-        print("Device: " + d.id());
-        if (flows == null | flows.isEmpty()) {
-            print(" %s", "No flows.");
-            return;
-        }
-        for (FlowRule f : flows) {
-            print(FMT, Long.toHexString(f.id().value()), f.state(), f.bytes(),
-                    f.packets(), f.lifeMillis(), f.priority());
-            print(SFMT, f.selector().criteria());
-            print(TFMT, f.treatment().instructions());
+    protected void printFlows(Device d, List<FlowEntry> flows) {
+        boolean empty = flows == null || flows.isEmpty();
+        print("deviceId=%s, flowRuleCount=%d", d.id(), empty ? 0 : flows.size());
+        if (!empty) {
+            for (FlowEntry f : flows) {
+                print(FMT, Long.toHexString(f.id().value()), f.state(), f.bytes(),
+                      f.packets(), f.life(), f.priority());
+                print(SFMT, f.selector().criteria());
+                print(TFMT, f.treatment().instructions());
+            }
         }
-
     }
 
 }

+package org.onlab.onos.cli.net;
+
+import org.apache.karaf.shell.console.Completer;
+import org.apache.karaf.shell.console.completer.StringsCompleter;
+import org.onlab.onos.cli.AbstractShellCommand;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentService;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.SortedSet;
+
+/**
+ * Intent ID completer.
+ */
+public class IntentIdCompleter implements Completer {
+    @Override
+    public int complete(String buffer, int cursor, List<String> candidates) {
+        // Delegate string completer
+        StringsCompleter delegate = new StringsCompleter();
+
+        // Fetch our service and feed it's offerings to the string completer
+        IntentService service = AbstractShellCommand.get(IntentService.class);
+        Iterator<Intent> it = service.getIntents().iterator();
+        SortedSet<String> strings = delegate.getStrings();
+        while (it.hasNext()) {
+            strings.add(it.next().id().toString());
+        }
+
+        // Now let the completer do the work for figuring out what to offer.
+        return delegate.complete(buffer, cursor, candidates);
+    }
+
+}

+package org.onlab.onos.cli.net;
+
+import org.apache.karaf.shell.commands.Argument;
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cli.AbstractShellCommand;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.IntentService;
+
+/**
+ * Removes host-to-host connectivity intent.
+ */
+@Command(scope = "onos", name = "remove-intent",
+         description = "Removes the specified intent")
+public class IntentRemoveCommand extends AbstractShellCommand {
+
+    @Argument(index = 0, name = "id", description = "Intent ID",
+              required = true, multiValued = false)
+    String id = null;
+
+    @Override
+    protected void execute() {
+        IntentService service = get(IntentService.class);
+
+        int radix = id.startsWith("0x") ? 16 : 10;
+        if (radix == 16) {
+            id = id.replaceFirst("0x", "");
+        }
+        IntentId intentId = new IntentId(Long.parseLong(id, radix));
+
+
+        Intent intent = service.getIntent(intentId);
+        if (intent != null) {
+            service.withdraw(intent);
+        }
+    }
+}

+package org.onlab.onos.cli.net;
+
+import org.apache.karaf.shell.commands.Command;
+import org.onlab.onos.cli.AbstractShellCommand;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentService;
+import org.onlab.onos.net.intent.IntentState;
+
+/**
+ * Lists the inventory of intents and their states.
+ */
+@Command(scope = "onos", name = "intents",
+         description = "Lists the inventory of intents and their states")
+public class IntentsListCommand extends AbstractShellCommand {
+
+    @Override
+    protected void execute() {
+        IntentService service = get(IntentService.class);
+        for (Intent intent : service.getIntents()) {
+            IntentState state = service.getIntentState(intent.id());
+            print("%s %s %s", intent.id(), state, intent);
+        }
+    }
+
+}

 package org.onlab.onos.cli.net;
 
+import org.apache.karaf.shell.commands.Argument;
 import org.apache.karaf.shell.commands.Command;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.Host;
@@ -7,28 +8,51 @@ import org.onlab.onos.net.device.DeviceAdminService;
 import org.onlab.onos.net.device.DeviceService;
 import org.onlab.onos.net.host.HostAdminService;
 import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentService;
+import org.onlab.onos.net.intent.IntentState;
 
 /**
- * Wipes-out the entire network information base, i.e. devices, links, hosts.
+ * Wipes-out the entire network information base, i.e. devices, links, hosts, intents.
  */
 @Command(scope = "onos", name = "wipe-out",
          description = "Wipes-out the entire network information base, i.e. devices, links, hosts")
 public class WipeOutCommand extends ClustersListCommand {
 
+    private static final String DISCLAIMER = "Delete everything please.";
+
+    @Argument(index = 0, name = "disclaimer", description = "Device ID",
+              required = false, multiValued = false)
+    String disclaimer = null;
+
     @Override
     protected void execute() {
+        if (disclaimer == null || !disclaimer.equals(DISCLAIMER)) {
+            print("I'm afraid I can't do that!\nPlease acknowledge with phrase: '%s'",
+                  DISCLAIMER);
+            return;
+        }
+
+        print("Wiping devices");
         DeviceAdminService deviceAdminService = get(DeviceAdminService.class);
         DeviceService deviceService = get(DeviceService.class);
         for (Device device : deviceService.getDevices()) {
             deviceAdminService.removeDevice(device.id());
         }
 
+        print("Wiping hosts");
         HostAdminService hostAdminService = get(HostAdminService.class);
         HostService hostService = get(HostService.class);
         for (Host host : hostService.getHosts()) {
             hostAdminService.removeHost(host.id());
         }
-    }
-
 
+        print("Wiping intents");
+        IntentService intentService = get(IntentService.class);
+        for (Intent intent : intentService.getIntents()) {
+            if (intentService.getIntentState(intent.id()) == IntentState.INSTALLED) {
+                intentService.withdraw(intent);
+            }
+        }
+    }
 }

@@ -2,6 +2,9 @@
 
     <command-bundle xmlns="http://karaf.apache.org/xmlns/shell/v1.1.0">
         <command>
+            <action class="org.onlab.onos.cli.SummaryCommand"/>
+        </command>
+        <command>
             <action class="org.onlab.onos.cli.NodesListCommand"/>
         </command>
         <command>
@@ -56,6 +59,30 @@
                 <ref component-id="deviceIdCompleter"/>
             </completers>
         </command>
+
+        <command>
+            <action class="org.onlab.onos.cli.net.IntentsListCommand"/>
+        </command>
+        <command>
+            <action class="org.onlab.onos.cli.net.IntentRemoveCommand"/>
+            <completers>
+                <ref component-id="intentIdCompleter"/>
+            </completers>
+        </command>
+        <command>
+            <action class="org.onlab.onos.cli.net.AddHostToHostIntentCommand"/>
+            <completers>
+                <ref component-id="hostIdCompleter"/>
+            </completers>
+        </command>
+        <command>
+            <action class="org.onlab.onos.cli.net.AddPointToPointIntentCommand"/>
+            <completers>
+                <ref component-id="connectPointCompleter"/>
+                <ref component-id="connectPointCompleter"/>
+            </completers>
+        </command>
+
         <command>
             <action class="org.onlab.onos.cli.net.ClustersListCommand"/>
         </command>
@@ -94,6 +121,8 @@
     <bean id="clusterIdCompleter" class="org.onlab.onos.cli.net.ClusterIdCompleter"/>
     <bean id="roleCompleter" class="org.onlab.onos.cli.net.RoleCompleter"/>
     <bean id="hostIdCompleter" class="org.onlab.onos.cli.net.HostIdCompleter"/>
+    <bean id="intentIdCompleter" class="org.onlab.onos.cli.net.IntentIdCompleter"/>
     <bean id="flowRuleStatusCompleter" class="org.onlab.onos.cli.net.FlowRuleStatusCompleter"/>
+    <bean id="connectPointCompleter" class="org.onlab.onos.cli.net.ConnectPointCompleter"/>
 
 </blueprint>

@@ -8,7 +8,7 @@ import java.util.concurrent.atomic.AtomicInteger;
  */
 public final class ApplicationId {
 
-    private static AtomicInteger idDispenser;
+    private static final AtomicInteger ID_DISPENCER = new AtomicInteger(1);
     private final Integer id;
 
     // Ban public construction
@@ -50,10 +50,7 @@ public final class ApplicationId {
      * @return app id
      */
     public static ApplicationId getAppId() {
-        if (ApplicationId.idDispenser == null) {
-            ApplicationId.idDispenser = new AtomicInteger(1);
-        }
-        return new ApplicationId(ApplicationId.idDispenser.getAndIncrement());
+        return new ApplicationId(ApplicationId.ID_DISPENCER.getAndIncrement());
     }
 
 }

+package org.onlab.onos;
+
+/**
+ * Service for interacting with the core system of the controller.
+ */
+public interface CoreService {
+
+    /**
+     * Returns the product version.
+     *
+     * @return product version
+     */
+    Version version();
+
+}

+package org.onlab.onos;
+
+import java.util.Objects;
+
+import static java.lang.Integer.parseInt;
+
+/**
+ * Representation of the product version.
+ */
+public final class Version {
+
+    public static final String FORMAT = "%d.%d.%d.%s";
+
+    private final int major;
+    private final int minor;
+    private final int patch;
+    private final String build;
+
+    private final String format;
+
+    // Creates a new version descriptor
+    private Version(int major, int minor, int patch, String build) {
+        this.major = major;
+        this.minor = minor;
+        this.patch = patch;
+        this.build = build;
+        this.format = String.format(FORMAT, major, minor, patch, build);
+    }
+
+
+    /**
+     * Creates a new version from the specified constituent numbers.
+     *
+     * @param major major version number
+     * @param minor minod version number
+     * @param patch version patch number
+     * @param build build string
+     * @return version descriptor
+     */
+    public static Version version(int major, int minor, int patch, String build) {
+        return new Version(major, minor, patch, build);
+    }
+
+    /**
+     * Creates a new version by parsing the specified string.
+     *
+     * @param string version string
+     * @return version descriptor
+     */
+    public static Version version(String string) {
+        String[] fields = string.split("[.-]");
+        return new Version(parseInt(fields[0]), parseInt(fields[1]),
+                           parseInt(fields[2]), fields[3]);
+    }
+
+    /**
+     * Returns the major version number.
+     *
+     * @return major version number
+     */
+    public int major() {
+        return major;
+    }
+
+    /**
+     * Returns the minor version number.
+     *
+     * @return minor version number
+     */
+    public int minor() {
+        return minor;
+    }
+
+    /**
+     * Returns the version patch number.
+     *
+     * @return patch number
+     */
+    public int patch() {
+        return patch;
+    }
+
+    /**
+     * Returns the version build string.
+     *
+     * @return build string
+     */
+    public String build() {
+        return build;
+    }
+
+    @Override
+    public String toString() {
+        return format;
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(format);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof Version) {
+            final Version other = (Version) obj;
+            return Objects.equals(this.format, other.format);
+        }
+        return false;
+    }
+}

@@ -17,6 +17,7 @@ public interface MastershipService {
      * Returns the role of the local node for the specified device, without
      * triggering master selection.
      *
+     * @param deviceId the the identifier of the device
      * @return role of the current node
      */
     MastershipRole getLocalRole(DeviceId deviceId);

+package org.onlab.onos.net;
+
+public final class AnnotationsUtil {
+
+    public static boolean isEqual(Annotations lhs, Annotations rhs) {
+        if (lhs == rhs) {
+            return true;
+        }
+        if (lhs == null || rhs == null) {
+            return false;
+        }
+
+        if (!lhs.keys().equals(rhs.keys())) {
+            return false;
+        }
+
+        for (String key : lhs.keys()) {
+            if (!lhs.value(key).equals(rhs.value(key))) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    // not to be instantiated
+    private AnnotationsUtil() {}
+}

@@ -47,7 +47,23 @@ public class ConnectPoint {
             return (DeviceId) elementId;
         }
         throw new IllegalStateException("Connection point not associated " +
-                "with an infrastructure device");
+                                                "with an infrastructure device");
+    }
+
+    /**
+     * Returns the identifier of the infrastructure device if the connection
+     * point belongs to a network element which is indeed an end-station host.
+     *
+     * @return network element identifier as a host identifier
+     * @throws java.lang.IllegalStateException if connection point is not
+     *                                         associated with a host
+     */
+    public HostId hostId() {
+        if (elementId instanceof HostId) {
+            return (HostId) elementId;
+        }
+        throw new IllegalStateException("Connection point not associated " +
+                                                "with an end-station host");
     }
 
     /**

@@ -73,31 +73,63 @@ public final class DefaultAnnotations implements SparseAnnotations {
     }
 
     /**
-     * Convert Annotations to DefaultAnnotations if needed and merges.
+     * Creates the union of two given SparseAnnotations.
+     * Unlike the {@link #merge(DefaultAnnotations, SparseAnnotations)} method,
+     * result will be {@link SparseAnnotations} instead of {@link Annotations}.
      *
-     * @see #merge(DefaultAnnotations, SparseAnnotations)
+     * A key tagged for removal will remain in the output SparseAnnotations,
+     * if the counterpart of the input does not contain the same key.
      *
      * @param annotations       base annotations
      * @param sparseAnnotations additional sparse annotations
      * @return combined annotations or the original base annotations if there
      * are not additional annotations
      */
-    public static DefaultAnnotations merge(Annotations annotations,
-                                    SparseAnnotations sparseAnnotations) {
+    public static SparseAnnotations union(SparseAnnotations annotations,
+                                          SparseAnnotations sparseAnnotations) {
+
+        if (sparseAnnotations == null || sparseAnnotations.keys().isEmpty()) {
+            return annotations;
+        }
+
+        final HashMap<String, String> newMap;
         if (annotations instanceof DefaultAnnotations) {
-            return merge((DefaultAnnotations) annotations, sparseAnnotations);
+            newMap = copy(((DefaultAnnotations) annotations).map);
+        } else {
+            newMap = new HashMap<>(annotations.keys().size() +
+                                   sparseAnnotations.keys().size());
+            putAllSparseAnnotations(newMap, annotations);
         }
 
-        DefaultAnnotations.Builder builder = DefaultAnnotations.builder();
-        for (String key : annotations.keys()) {
-            builder.set(key, annotations.value(key));
+        putAllSparseAnnotations(newMap, sparseAnnotations);
+        return new DefaultAnnotations(newMap);
+    }
+
+    // adds the key-values contained in sparseAnnotations to
+    // newMap, if sparseAnnotations had a key tagged for removal,
+    // and corresponding key exist in newMap, entry will be removed.
+    // if corresponding key does not exist, removal tag will be added to
+    // the newMap.
+    private static void putAllSparseAnnotations(
+                            final HashMap<String, String> newMap,
+                            SparseAnnotations sparseAnnotations) {
+
+        for (String key : sparseAnnotations.keys()) {
+            if (sparseAnnotations.isRemoved(key)) {
+                if (newMap.containsKey(key)) {
+                    newMap.remove(key);
+                } else {
+                    newMap.put(key, Builder.REMOVED);
+                }
+            } else {
+                String value = sparseAnnotations.value(key);
+                newMap.put(key, value);
+            }
         }
-        return merge(builder.build(), sparseAnnotations);
     }
 
     @Override
     public Set<String> keys() {
-        // TODO: unmodifiable to be removed after switching to ImmutableMap;
         return Collections.unmodifiableSet(map.keySet());
     }
 
@@ -115,7 +147,7 @@ public final class DefaultAnnotations implements SparseAnnotations {
     @SuppressWarnings("unchecked")
     private static HashMap<String, String> copy(Map<String, String> original) {
         if (original instanceof HashMap) {
-            return (HashMap) ((HashMap) original).clone();
+            return (HashMap<String, String>) ((HashMap<?, ?>) original).clone();
         }
         throw new IllegalArgumentException("Expecting HashMap instance");
     }

@@ -3,6 +3,7 @@ package org.onlab.onos.net;
 import org.onlab.onos.net.provider.ProviderId;
 
 import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 /**
  * Default edge link model implementation.
@@ -18,9 +19,9 @@ public class DefaultEdgeLink extends DefaultLink implements EdgeLink {
      * @param providerId   provider identity
      * @param hostPoint    host-side connection point
      * @param hostLocation location where host attaches to the network
-     * @param isIngress    true to indicated host-to-network direction; false
+     * @param isIngress    true to indicate host-to-network direction; false
      *                     for network-to-host direction
-     * @param annotations optional key/value annotations
+     * @param annotations  optional key/value annotations
      */
     public DefaultEdgeLink(ProviderId providerId, ConnectPoint hostPoint,
                            HostLocation hostLocation, boolean isIngress,
@@ -42,4 +43,24 @@ public class DefaultEdgeLink extends DefaultLink implements EdgeLink {
     public HostLocation hostLocation() {
         return hostLocation;
     }
+
+    /**
+     * Creates a phantom edge link, to an unspecified end-station. This link
+     * does not represent any actually discovered link stored in the system.
+     *
+     * @param edgePort  network edge port
+     * @param isIngress true to indicate host-to-network direction; false
+     *                  for network-to-host direction
+     * @return new phantom edge link
+     */
+    public static DefaultEdgeLink createEdgeLink(ConnectPoint edgePort,
+                                                 boolean isIngress) {
+        checkNotNull(edgePort, "Edge port cannot be null");
+        HostLocation location = (edgePort instanceof HostLocation) ?
+                (HostLocation) edgePort : new HostLocation(edgePort, 0);
+        return new DefaultEdgeLink(ProviderId.NONE,
+                                   new ConnectPoint(HostId.NONE, PortNumber.P0),
+                                   location, isIngress);
+    }
+
 }

@@ -10,6 +10,14 @@ import java.net.URI;
  */
 public final class HostId extends ElementId {
 
+    private static final String NIC = "nic";
+
+    /**
+     * Represents either no host, or an unspecified host; used for creating
+     * open ingress/egress edge links.
+     */
+    public static final HostId NONE = hostId(NIC + ":none-0");
+
     // Public construction is prohibited
     private HostId(URI uri) {
         super(uri);
@@ -43,8 +51,7 @@ public final class HostId extends ElementId {
      * @return host identifier
      */
     public static HostId hostId(MacAddress mac, VlanId vlanId) {
-        // FIXME: use more efficient means of encoding
-        return hostId("nic" + ":" + mac + "-" + vlanId);
+        return hostId(NIC + ":" + mac + "-" + vlanId);
     }
 
     /**

@@ -22,6 +22,17 @@ public class HostLocation extends ConnectPoint {
     }
 
     /**
+     * Creates a new host location derived from the supplied connection point.
+     *
+     * @param connectPoint connection point
+     * @param time         time when detected, in millis since start of epoch
+     */
+    public HostLocation(ConnectPoint connectPoint, long time) {
+        super(connectPoint.deviceId(), connectPoint.port());
+        this.time = time;
+    }
+
+    /**
      * Returns the time when the location was established, given in
      * milliseconds since start of epoch.
      *

@@ -6,6 +6,7 @@ import com.google.common.base.MoreObjects;
 
 // TODO Consider renaming.
 // it's an identifier for a Link, but it's not ElementId, so not using LinkId.
+
 /**
  * Immutable representation of a link identity.
  */
@@ -43,6 +44,15 @@ public class LinkKey {
         this.dst = dst;
     }
 
+    /**
+     * Creates a link identifier for the specified link.
+     *
+     * @param link link descriptor
+     */
+    public LinkKey(Link link) {
+        this(link.src(), link.dst());
+    }
+
     @Override
     public int hashCode() {
         return Objects.hash(src(), dst);

@@ -9,6 +9,8 @@ import com.google.common.primitives.UnsignedLongs;
  */
 public final class PortNumber {
 
+    public static final PortNumber P0 = portNumber(0);
+
     // TODO: revisit the max and the logical port value assignments
 
     private static final long MAX_NUMBER = (2L * Integer.MAX_VALUE) + 1;

@@ -96,4 +96,13 @@ public class DefaultDeviceDescription extends AbstractDescription
                 .toString();
     }
 
+    // default constructor for serialization
+    private DefaultDeviceDescription() {
+        this.uri = null;
+        this.type = null;
+        this.manufacturer = null;
+        this.hwVersion = null;
+        this.swVersion = null;
+        this.serialNumber = null;
+    }
 }

@@ -48,4 +48,9 @@ public class DefaultPortDescription extends AbstractDescription
         return isEnabled;
     }
 
+    // default constructor for serialization
+    private DefaultPortDescription() {
+        this.number = null;
+        this.isEnabled = false;
+    }
 }

+package org.onlab.onos.net.flow;
+
+public class CompletedBatchOperation {
+
+
+}

+package org.onlab.onos.net.flow;
+
+import static com.google.common.base.MoreObjects.toStringHelper;
+import static org.slf4j.LoggerFactory.getLogger;
+
+import org.onlab.onos.net.DeviceId;
+import org.slf4j.Logger;
+
+public class DefaultFlowEntry extends DefaultFlowRule implements FlowEntry {
+
+    private final Logger log = getLogger(getClass());
+
+    private long life;
+    private long packets;
+    private long bytes;
+    private FlowEntryState state;
+
+    private long lastSeen = -1;
+
+
+    public DefaultFlowEntry(DeviceId deviceId, TrafficSelector selector,
+            TrafficTreatment treatment, int priority, FlowEntryState state,
+            long life, long packets, long bytes, long flowId,
+            int timeout) {
+        super(deviceId, selector, treatment, priority, flowId, timeout);
+        this.state = state;
+        this.life = life;
+        this.packets = packets;
+        this.bytes = bytes;
+        this.lastSeen = System.currentTimeMillis();
+    }
+
+    public DefaultFlowEntry(FlowRule rule, FlowEntryState state,
+            long life, long packets, long bytes) {
+        super(rule);
+        this.state = state;
+        this.life = life;
+        this.packets = packets;
+        this.bytes = bytes;
+        this.lastSeen = System.currentTimeMillis();
+    }
+
+    public DefaultFlowEntry(FlowRule rule) {
+        super(rule);
+        this.state = FlowEntryState.PENDING_ADD;
+        this.life = 0;
+        this.packets = 0;
+        this.bytes = 0;
+        this.lastSeen = System.currentTimeMillis();
+    }
+
+    @Override
+    public long life() {
+        return life;
+    }
+
+    @Override
+    public long packets() {
+        return packets;
+    }
+
+    @Override
+    public long bytes() {
+        return bytes;
+    }
+
+    @Override
+    public FlowEntryState state() {
+        return this.state;
+    }
+
+    @Override
+    public long lastSeen() {
+        return lastSeen;
+    }
+
+    @Override
+    public void setLastSeen() {
+        this.lastSeen = System.currentTimeMillis();
+    }
+
+    @Override
+    public void setState(FlowEntryState newState) {
+        this.state = newState;
+    }
+
+    @Override
+    public void setLife(long life) {
+        this.life = life;
+    }
+
+    @Override
+    public void setPackets(long packets) {
+        this.packets = packets;
+    }
+
+    @Override
+    public void setBytes(long bytes) {
+        this.bytes = bytes;
+    }
+
+    @Override
+    public String toString() {
+        return toStringHelper(this)
+                .add("rule", super.toString())
+                .add("state", state)
+                .toString();
+    }
+
+
+}

@@ -18,10 +18,6 @@ public class DefaultFlowRule implements FlowRule {
     private final TrafficSelector selector;
     private final TrafficTreatment treatment;
     private final long created;
-    private final long life;
-    private final long packets;
-    private final long bytes;
-    private final FlowRuleState state;
 
     private final FlowId id;
 
@@ -29,73 +25,50 @@ public class DefaultFlowRule implements FlowRule {
 
     private final int timeout;
 
+
     public DefaultFlowRule(DeviceId deviceId, TrafficSelector selector,
-            TrafficTreatment treatment, int priority, FlowRuleState state,
-            long life, long packets, long bytes, long flowId, boolean expired,
+            TrafficTreatment treatment, int priority, long flowId,
             int timeout) {
         this.deviceId = deviceId;
         this.priority = priority;
         this.selector = selector;
         this.treatment = treatment;
-        this.state = state;
+        this.timeout = timeout;
+        this.created = System.currentTimeMillis();
+
         this.appId = ApplicationId.valueOf((int) (flowId >> 32));
         this.id = FlowId.valueOf(flowId);
-        this.life = life;
-        this.packets = packets;
-        this.bytes = bytes;
-        this.created = System.currentTimeMillis();
-        this.timeout = timeout;
     }
 
     public DefaultFlowRule(DeviceId deviceId, TrafficSelector selector,
             TrafficTreatment treatement, int priority, ApplicationId appId,
             int timeout) {
-        this(deviceId, selector, treatement, priority,
-                FlowRuleState.CREATED, appId, timeout);
-    }
 
-    public DefaultFlowRule(FlowRule rule, FlowRuleState state) {
-        this(rule.deviceId(), rule.selector(), rule.treatment(),
-                rule.priority(), state, rule.id(), rule.appId(),
-                rule.timeout());
-    }
+        if (priority < FlowRule.MIN_PRIORITY) {
+            throw new IllegalArgumentException("Priority cannot be less than " + MIN_PRIORITY);
+        }
 
-    private DefaultFlowRule(DeviceId deviceId,
-            TrafficSelector selector, TrafficTreatment treatment,
-            int priority, FlowRuleState state, ApplicationId appId,
-            int timeout) {
         this.deviceId = deviceId;
         this.priority = priority;
         this.selector = selector;
-        this.treatment = treatment;
-        this.state = state;
-        this.life = 0;
-        this.packets = 0;
-        this.bytes = 0;
+        this.treatment = treatement;
         this.appId = appId;
-
         this.timeout = timeout;
+        this.created = System.currentTimeMillis();
 
         this.id = FlowId.valueOf((((long) appId().id()) << 32) | (this.hash() & 0xffffffffL));
-        this.created = System.currentTimeMillis();
     }
 
-    private DefaultFlowRule(DeviceId deviceId,
-            TrafficSelector selector, TrafficTreatment treatment,
-            int priority, FlowRuleState state, FlowId flowId, ApplicationId appId,
-            int timeout) {
-        this.deviceId = deviceId;
-        this.priority = priority;
-        this.selector = selector;
-        this.treatment = treatment;
-        this.state = state;
-        this.life = 0;
-        this.packets = 0;
-        this.bytes = 0;
-        this.appId = appId;
-        this.id = flowId;
-        this.timeout = timeout;
+    public DefaultFlowRule(FlowRule rule) {
+        this.deviceId = rule.deviceId();
+        this.priority = rule.priority();
+        this.selector = rule.selector();
+        this.treatment = rule.treatment();
+        this.appId = rule.appId();
+        this.id = rule.id();
+        this.timeout = rule.timeout();
         this.created = System.currentTimeMillis();
+
     }
 
 
@@ -129,26 +102,6 @@ public class DefaultFlowRule implements FlowRule {
         return treatment;
     }
 
-    @Override
-    public long lifeMillis() {
-        return life;
-    }
-
-    @Override
-    public long packets() {
-        return packets;
-    }
-
-    @Override
-    public long bytes() {
-        return bytes;
-    }
-
-    @Override
-    public FlowRuleState state() {
-        return this.state;
-    }
-
 
     @Override
     /*
@@ -162,7 +115,7 @@ public class DefaultFlowRule implements FlowRule {
     }
 
     public int hash() {
-        return Objects.hash(deviceId, selector, id);
+        return Objects.hash(deviceId, selector, treatment);
     }
 
     @Override
@@ -179,7 +132,7 @@ public class DefaultFlowRule implements FlowRule {
         if (obj instanceof DefaultFlowRule) {
             DefaultFlowRule that = (DefaultFlowRule) obj;
             return Objects.equals(deviceId, that.deviceId) &&
-                    //Objects.equals(id, that.id) &&
+                    Objects.equals(id, that.id) &&
                     Objects.equals(priority, that.priority) &&
                     Objects.equals(selector, that.selector);
 
@@ -190,19 +143,18 @@ public class DefaultFlowRule implements FlowRule {
     @Override
     public String toString() {
         return toStringHelper(this)
-                .add("id", id)
+                .add("id", Long.toHexString(id.value()))
                 .add("deviceId", deviceId)
                 .add("priority", priority)
                 .add("selector", selector.criteria())
                 .add("treatment", treatment == null ? "N/A" : treatment.instructions())
                 .add("created", created)
-                .add("state", state)
                 .toString();
     }
 
     @Override
     public int timeout() {
-        return timeout > MAX_TIMEOUT ? MAX_TIMEOUT : this.timeout;
+        return timeout;
     }
 
 }

 package org.onlab.onos.net.flow;
 
-import static org.slf4j.LoggerFactory.getLogger;
-
-import java.util.Collections;
-import java.util.HashSet;
-import java.util.Objects;
-import java.util.Set;
-
+import com.google.common.collect.ImmutableSet;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.flow.criteria.Criteria;
 import org.onlab.onos.net.flow.criteria.Criterion;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
-import org.slf4j.Logger;
 
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Default traffic selector implementation.
+ */
 public final class DefaultTrafficSelector implements TrafficSelector {
 
-    private final Set<Criterion> selector;
+    private final Set<Criterion> criteria;
 
-    private DefaultTrafficSelector(Set<Criterion> selector) {
-        this.selector = Collections.unmodifiableSet(selector);
+    /**
+     * Creates a new traffic selector with the specified criteria.
+     *
+     * @param criteria criteria
+     */
+    private DefaultTrafficSelector(Set<Criterion> criteria) {
+        this.criteria = Collections.unmodifiableSet(criteria);
     }
 
     @Override
     public Set<Criterion> criteria() {
-        return selector;
+        return criteria;
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(selector);
+        return Objects.hash(criteria);
     }
 
     @Override
@@ -40,23 +47,50 @@ public final class DefaultTrafficSelector implements TrafficSelector {
         }
         if (obj instanceof DefaultTrafficSelector) {
             DefaultTrafficSelector that = (DefaultTrafficSelector) obj;
-            return Objects.equals(selector, that.selector);
+            return Objects.equals(criteria, that.criteria);
 
         }
         return false;
     }
 
+    /**
+     * Returns a new traffic selector builder.
+     *
+     * @return traffic selector builder
+     */
+    public static TrafficSelector.Builder builder() {
+        return new Builder();
+    }
 
+    /**
+     * Returns a new traffic selector builder primed to produce entities
+     * patterned after the supplied selector.
+     *
+     * @return traffic selector builder
+     */
+    public static TrafficSelector.Builder builder(TrafficSelector selector) {
+        return new Builder(selector);
+    }
 
-    public static class Builder implements TrafficSelector.Builder {
+    /**
+     * Builder of traffic selector entities.
+     */
+    public static final class Builder implements TrafficSelector.Builder {
 
-        private final Logger log = getLogger(getClass());
+        private final Map<Criterion.Type, Criterion> selector = new HashMap<>();
 
-        private final Set<Criterion> selector = new HashSet<>();
+        private Builder() {
+        }
+
+        private Builder(TrafficSelector selector) {
+            for (Criterion c : selector.criteria()) {
+                add(c);
+            }
+        }
 
         @Override
         public Builder add(Criterion criterion) {
-            selector.add(criterion);
+            selector.put(criterion.type(), criterion);
             return this;
         }
 
@@ -107,7 +141,7 @@ public final class DefaultTrafficSelector implements TrafficSelector {
 
         @Override
         public TrafficSelector build() {
-            return new DefaultTrafficSelector(selector);
+            return new DefaultTrafficSelector(ImmutableSet.copyOf(selector.values()));
         }
 
     }

@@ -5,6 +5,7 @@ import static org.slf4j.LoggerFactory.getLogger;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Objects;
 
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.flow.instructions.Instruction;
@@ -14,10 +15,18 @@ import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.slf4j.Logger;
 
+/**
+ * Default traffic treatment implementation.
+ */
 public final class DefaultTrafficTreatment implements TrafficTreatment {
 
     private final List<Instruction> instructions;
 
+    /**
+     * Creates a new traffic treatment from the specified list of instructions.
+     *
+     * @param instructions treatment instructions
+     */
     private DefaultTrafficTreatment(List<Instruction> instructions) {
         this.instructions = Collections.unmodifiableList(instructions);
     }
@@ -28,12 +37,38 @@ public final class DefaultTrafficTreatment implements TrafficTreatment {
     }
 
     /**
-     * Builds a list of treatments following the following order.
-     * Modifications -> Group -> Output (including drop)
+     * Returns a new traffic treatment builder.
      *
+     * @return traffic treatment builder
      */
+    public static TrafficTreatment.Builder builder() {
+        return new Builder();
+    }
+
+    //FIXME: Order of instructions may affect hashcode
+    @Override
+    public int hashCode() {
+        return Objects.hash(instructions);
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj instanceof DefaultTrafficTreatment) {
+            DefaultTrafficTreatment that = (DefaultTrafficTreatment) obj;
+            return  Objects.equals(instructions, that.instructions);
 
-    public static class Builder implements TrafficTreatment.Builder {
+        }
+        return false;
+    }
+
+    /**
+     * Builds a list of treatments following the following order.
+     * Modifications -> Group -> Output (including drop)
+     */
+    public static final class Builder implements TrafficTreatment.Builder {
 
         private final Logger log = getLogger(getClass());
 
@@ -47,27 +82,32 @@ public final class DefaultTrafficTreatment implements TrafficTreatment {
         // TODO: should be a list of instructions based on modification objects
         List<Instruction> modifications = new LinkedList<>();
 
+        // Creates a new builder
+        private Builder() {
+        }
+
+        @Override
         public Builder add(Instruction instruction) {
             if (drop) {
                 return this;
             }
             switch (instruction.type()) {
-            case DROP:
-                drop = true;
-                break;
-            case OUTPUT:
-                outputs.add(instruction);
-                break;
-            case L2MODIFICATION:
-            case L3MODIFICATION:
-                // TODO: enforce modification order if any
-                modifications.add(instruction);
-                break;
-            case GROUP:
-                groups.add(instruction);
-                break;
-            default:
-                log.warn("Unknown instruction type {}", instruction.type());
+                case DROP:
+                    drop = true;
+                    break;
+                case OUTPUT:
+                    outputs.add(instruction);
+                    break;
+                case L2MODIFICATION:
+                case L3MODIFICATION:
+                    // TODO: enforce modification order if any
+                    modifications.add(instruction);
+                    break;
+                case GROUP:
+                    groups.add(instruction);
+                    break;
+                default:
+                    log.warn("Unknown instruction type {}", instruction.type());
             }
             return this;
         }

+package org.onlab.onos.net.flow;
+
+
+/**
+ * Represents a generalized match & action pair to be applied to
+ * an infrastucture device.
+ */
+public interface FlowEntry extends FlowRule {
+
+
+    public enum FlowEntryState {
+
+        /**
+         * Indicates that this rule has been submitted for addition.
+         * Not necessarily in  the flow table.
+         */
+        PENDING_ADD,
+
+        /**
+         * Rule has been added which means it is in the flow table.
+         */
+        ADDED,
+
+        /**
+         * Flow has been marked for removal, might still be in flow table.
+         */
+        PENDING_REMOVE,
+
+        /**
+         * Flow has been removed from flow table and can be purged.
+         */
+        REMOVED
+    }
+
+    /**
+     * Returns the flow entry state.
+     *
+     * @return flow entry state
+     */
+    FlowEntryState state();
+
+    /**
+     * Returns the number of milliseconds this flow rule has been applied.
+     *
+     * @return number of millis
+     */
+    long life();
+
+    /**
+     * Returns the number of packets this flow rule has matched.
+     *
+     * @return number of packets
+     */
+    long packets();
+
+    /**
+     * Returns the number of bytes this flow rule has matched.
+     *
+     * @return number of bytes
+     */
+    long bytes();
+
+    /**
+     * When this flow entry was last deemed active.
+     * @return epoch time of last activity
+     */
+    long lastSeen();
+
+    /**
+     * Sets the last active epoch time.
+     */
+    void setLastSeen();
+
+    /**
+     * Sets the new state for this entry.
+     * @param newState new flow entry state.
+     */
+    void setState(FlowEntryState newState);
+
+    /**
+     * Sets how long this entry has been entered in the system.
+     * @param life epoch time
+     */
+    void setLife(long life);
+
+    /**
+     * Number of packets seen by this entry.
+     * @param packets a long value
+     */
+    void setPackets(long packets);
+
+    /**
+     * Number of bytes seen by this rule.
+     * @param bytes a long value
+     */
+    void setBytes(long bytes);
+
+}

@@ -26,6 +26,9 @@ public final class FlowId {
         if (this == obj) {
             return true;
         }
+        if (obj == null) {
+            return false;
+        }
         if (obj.getClass()  == this.getClass()) {
             FlowId that = (FlowId) obj;
             return Objects.equal(this.flowid, that.flowid);

@@ -2,49 +2,16 @@ package org.onlab.onos.net.flow;
 
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.intent.BatchOperationTarget;
 
 /**
  * Represents a generalized match & action pair to be applied to
  * an infrastucture device.
  */
-public interface FlowRule {
+public interface FlowRule extends BatchOperationTarget {
 
     static final int MAX_TIMEOUT = 60;
-
-    public enum FlowRuleState {
-        /**
-         * Indicates that this rule has been created.
-         */
-        CREATED,
-
-        /**
-         * Indicates that this rule has been submitted for addition.
-         * Not necessarily in  the flow table.
-         */
-        PENDING_ADD,
-
-        /**
-         * Rule has been added which means it is in the flow table.
-         */
-        ADDED,
-
-        /**
-         * Flow has been marked for removal, might still be in flow table.
-         */
-        PENDING_REMOVE,
-
-        /**
-         * Flow has been removed from flow table and can be purged.
-         */
-        REMOVED
-    }
-
-    /**
-     * Returns the flow rule state.
-     *
-     * @return flow rule state
-     */
-    FlowRuleState state();
+    static final int MIN_PRIORITY = 0;
 
     //TODO: build cookie value
     /**
@@ -92,27 +59,6 @@ public interface FlowRule {
     TrafficTreatment treatment();
 
     /**
-     * Returns the number of milliseconds this flow rule has been applied.
-     *
-     * @return number of millis
-     */
-    long lifeMillis();
-
-    /**
-     * Returns the number of packets this flow rule has matched.
-     *
-     * @return number of packets
-     */
-    long packets();
-
-    /**
-     * Returns the number of bytes this flow rule has matched.
-     *
-     * @return number of bytes
-     */
-    long bytes();
-
-    /**
      * Returns the timeout for this flow requested by an application.
      * @return integer value of the timeout
      */

+package org.onlab.onos.net.flow;
+
+import org.onlab.onos.net.flow.FlowRuleBatchEntry.FlowRuleOperation;
+import org.onlab.onos.net.intent.BatchOperationEntry;
+
+
+public class FlowRuleBatchEntry
+        extends BatchOperationEntry<FlowRuleOperation, FlowRule> {
+
+    public FlowRuleBatchEntry(FlowRuleOperation operator, FlowRule target) {
+        super(operator, target);
+    }
+
+    public enum FlowRuleOperation {
+        ADD,
+        REMOVE,
+        MODIFY
+    }
+
+}

+package org.onlab.onos.net.flow;
+
+import java.util.Collection;
+
+import org.onlab.onos.net.intent.BatchOperation;
+
+public class FlowRuleBatchOperation
+    extends BatchOperation<FlowRuleBatchEntry> {
+
+    public FlowRuleBatchOperation(Collection<FlowRuleBatchEntry> operations) {
+        super(operations);
+    }
+}

 package org.onlab.onos.net.flow;
 
+import java.util.concurrent.Future;
+
 import org.onlab.onos.ApplicationId;
+import org.onlab.onos.net.intent.BatchOperation;
 import org.onlab.onos.net.provider.Provider;
 
 /**
@@ -34,4 +37,6 @@ public interface FlowRuleProvider extends Provider {
      */
     void removeRulesById(ApplicationId id, FlowRule... flowRules);
 
+    Future<Void> executeBatch(BatchOperation<FlowRuleBatchEntry> batch);
+
 }

@@ -14,7 +14,7 @@ public interface FlowRuleProviderService extends ProviderService<FlowRuleProvide
      *
      * @param flowRule information about the removed flow
      */
-    void flowRemoved(FlowRule flowRule);
+    void flowRemoved(FlowEntry flowEntry);
 
     /**
      * Pushes the collection of flow entries currently applied on the given
@@ -22,7 +22,7 @@ public interface FlowRuleProviderService extends ProviderService<FlowRuleProvide
      *
      * @param flowRules collection of flow rules
      */
-    void pushFlowMetrics(DeviceId deviceId, Iterable<FlowRule> flowRules);
+    void pushFlowMetrics(DeviceId deviceId, Iterable<FlowEntry> flowEntries);
 
 
 

 package org.onlab.onos.net.flow;
 
+import java.util.concurrent.Future;
+
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.net.DeviceId;
 
@@ -13,6 +15,13 @@ import org.onlab.onos.net.DeviceId;
 public interface FlowRuleService {
 
     /**
+     * Returns the number of flow rules in the system.
+     *
+     * @return flow rule count
+     */
+    int getFlowRuleCount();
+
+    /**
      * Returns the collection of flow entries applied on the specified device.
      * This will include flow rules which may not yet have been applied to
      * the device.
@@ -20,7 +29,7 @@ public interface FlowRuleService {
      * @param deviceId device identifier
      * @return collection of flow rules
      */
-    Iterable<FlowRule> getFlowEntries(DeviceId deviceId);
+    Iterable<FlowEntry> getFlowEntries(DeviceId deviceId);
 
     // TODO: add createFlowRule factory method and execute operations method
 
@@ -60,6 +69,13 @@ public interface FlowRuleService {
     Iterable<FlowRule> getFlowRulesById(ApplicationId id);
 
     /**
+     * Applies a batch operation of FlowRules.
+     *
+     * @return future indicating the state of the batch operation
+     */
+    Future<CompletedBatchOperation> applyBatch(FlowRuleBatchOperation batch);
+
+    /**
      * Adds the specified flow rule listener.
      *
      * @param listener flow rule listener
@@ -72,7 +88,4 @@ public interface FlowRuleService {
      * @param listener flow rule listener
      */
     void removeListener(FlowRuleListener listener);
-
-
-
 }

@@ -10,11 +10,19 @@ import org.onlab.onos.store.Store;
 public interface FlowRuleStore extends Store<FlowRuleEvent, FlowRuleStoreDelegate> {
 
     /**
+     * Returns the number of flow rule in the store.
+     *
+     * @return number of flow rules
+     */
+    int getFlowRuleCount();
+
+    /**
      * Returns the stored flow.
+     *
      * @param rule the rule to look for
      * @return a flow rule
      */
-    FlowRule getFlowRule(FlowRule rule);
+    FlowEntry getFlowEntry(FlowRule rule);
 
     /**
      * Returns the flow entries associated with a device.
@@ -22,7 +30,7 @@ public interface FlowRuleStore extends Store<FlowRuleEvent, FlowRuleStoreDelegat
      * @param deviceId the device ID
      * @return the flow entries
      */
-    Iterable<FlowRule> getFlowEntries(DeviceId deviceId);
+    Iterable<FlowEntry> getFlowEntries(DeviceId deviceId);
 
     /**
      * Returns the flow entries associated with an application.
@@ -30,7 +38,7 @@ public interface FlowRuleStore extends Store<FlowRuleEvent, FlowRuleStoreDelegat
      * @param appId the application id
      * @return the flow entries
      */
-    Iterable<FlowRule> getFlowEntriesByAppId(ApplicationId appId);
+    Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId);
 
     /**
      * Stores a new flow rule without generating events.
@@ -40,7 +48,8 @@ public interface FlowRuleStore extends Store<FlowRuleEvent, FlowRuleStoreDelegat
     void storeFlowRule(FlowRule rule);
 
     /**
-     * Deletes a flow rule without generating events.
+     * Marks a flow rule for deletion. Actual deletion will occur
+     * when the provider indicates that the flow has been removed.
      *
      * @param rule the flow rule to delete
      */
@@ -52,12 +61,11 @@ public interface FlowRuleStore extends Store<FlowRuleEvent, FlowRuleStoreDelegat
      * @param rule the flow rule to add or update
      * @return flow_added event, or null if just an update
      */
-    FlowRuleEvent addOrUpdateFlowRule(FlowRule rule);
+    FlowRuleEvent addOrUpdateFlowRule(FlowEntry rule);
 
     /**
-     * @param rule the flow rule to remove
+     * @param rule the flow entry to remove
      * @return flow_removed event, or null if nothing removed
      */
-    FlowRuleEvent removeFlowRule(FlowRule rule);
-
+    FlowRuleEvent removeFlowRule(FlowEntry rule);
 }

@@ -3,6 +3,8 @@ package org.onlab.onos.net.flow.instructions;
 import static com.google.common.base.MoreObjects.toStringHelper;
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.util.Objects;
+
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.flow.instructions.L2ModificationInstruction.L2SubType;
 import org.onlab.onos.net.flow.instructions.L2ModificationInstruction.ModEtherInstruction;
@@ -117,6 +119,24 @@ public final class Instructions {
             return toStringHelper(type()).toString();
 
         }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(type());
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj instanceof DropInstruction) {
+                DropInstruction that = (DropInstruction) obj;
+                return Objects.equals(type(), that.type());
+
+            }
+            return false;
+        }
     }
 
 
@@ -140,6 +160,26 @@ public final class Instructions {
             return toStringHelper(type().toString())
                     .add("port", port).toString();
         }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(port, type());
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj instanceof OutputInstruction) {
+                OutputInstruction that = (OutputInstruction) obj;
+                        Objects.equals(port, that.port);
+
+            }
+            return false;
+        }
     }
 
 }
+
+

@@ -2,6 +2,8 @@ package org.onlab.onos.net.flow.instructions;
 
 import static com.google.common.base.MoreObjects.toStringHelper;
 
+import java.util.Objects;
+
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 
@@ -74,6 +76,25 @@ public abstract class L2ModificationInstruction implements Instruction {
                     .add("mac", mac).toString();
         }
 
+        @Override
+        public int hashCode() {
+            return Objects.hash(mac, subtype);
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj instanceof ModEtherInstruction) {
+                ModEtherInstruction that = (ModEtherInstruction) obj;
+                return  Objects.equals(mac, that.mac) &&
+                        Objects.equals(subtype, that.subtype);
+
+            }
+            return false;
+        }
+
 
     }
 
@@ -103,6 +124,25 @@ public abstract class L2ModificationInstruction implements Instruction {
                     .add("id", vlanId).toString();
         }
 
+        @Override
+        public int hashCode() {
+            return Objects.hash(vlanId, subtype());
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj instanceof ModVlanIdInstruction) {
+                ModVlanIdInstruction that = (ModVlanIdInstruction) obj;
+                return  Objects.equals(vlanId, that.vlanId);
+
+            }
+            return false;
+        }
+
+
     }
 
     /**
@@ -131,6 +171,24 @@ public abstract class L2ModificationInstruction implements Instruction {
                     .add("pcp", Long.toHexString(vlanPcp)).toString();
         }
 
+        @Override
+        public int hashCode() {
+            return Objects.hash(vlanPcp, subtype());
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj instanceof ModVlanPcpInstruction) {
+                ModVlanPcpInstruction that = (ModVlanPcpInstruction) obj;
+                return  Objects.equals(vlanPcp, that.vlanPcp);
+
+            }
+            return false;
+        }
+
     }
 
 

@@ -2,6 +2,8 @@ package org.onlab.onos.net.flow.instructions;
 
 import static com.google.common.base.MoreObjects.toStringHelper;
 
+import java.util.Objects;
+
 import org.onlab.packet.IpPrefix;
 
 /**
@@ -66,5 +68,23 @@ public abstract class L3ModificationInstruction implements Instruction {
                     .add("ip", ip).toString();
         }
 
+        @Override
+        public int hashCode() {
+            return Objects.hash(ip, subtype());
+        }
+
+        @Override
+        public boolean equals(Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (obj instanceof ModIPInstruction) {
+                ModIPInstruction that = (ModIPInstruction) obj;
+                return  Objects.equals(ip, that.ip);
+
+            }
+            return false;
+        }
+
     }
 }

@@ -24,7 +24,7 @@ public abstract class AbstractIntent implements Intent {
     }
 
     @Override
-    public IntentId getId() {
+    public IntentId id() {
         return id;
     }
 

@@ -3,6 +3,7 @@ package org.onlab.onos.net.intent;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
@@ -11,11 +12,11 @@ import java.util.List;
  * A list of BatchOperationEntry.
  *
  * @param <T> the enum of operators <br>
- *        This enum must be defined in each sub-classes.
- *
+ *            This enum must be defined in each sub-classes.
  */
 public abstract class BatchOperation<T extends BatchOperationEntry<?, ?>> {
-    private List<T> ops;
+
+    private final List<T> ops;
 
     /**
      * Creates new {@link BatchOperation} object.
@@ -30,7 +31,7 @@ public abstract class BatchOperation<T extends BatchOperationEntry<?, ?>> {
      *
      * @param batchOperations the list of batch operation entries.
      */
-    public BatchOperation(List<T> batchOperations) {
+    public BatchOperation(Collection<T> batchOperations) {
         ops = new LinkedList<>(checkNotNull(batchOperations));
     }
 
@@ -61,6 +62,10 @@ public abstract class BatchOperation<T extends BatchOperationEntry<?, ?>> {
 
     /**
      * Adds an operation.
+     * FIXME: Brian promises that the Intent Framework
+     * will not modify the batch operation after it has submitted it.
+     * Ali would prefer immutablity, but trusts brian for better or
+     * for worse.
      *
      * @param entry the operation to be added
      * @return this object if succeeded, null otherwise

@@ -15,14 +15,7 @@ public class BatchOperationEntry<T extends Enum<?>, U extends BatchOperationTarg
     private final T operator;
     private final U target;
 
-    /**
-     * Default constructor for serializer.
-     */
-    @Deprecated
-    protected BatchOperationEntry() {
-        this.operator = null;
-        this.target = null;
-    }
+
 
     /**
      * Constructs new instance for the entry of the BatchOperation.

 package org.onlab.onos.net.intent;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
+import com.google.common.base.Objects;
 import org.onlab.onos.net.flow.TrafficSelector;
 import org.onlab.onos.net.flow.TrafficTreatment;
 
-import com.google.common.base.Objects;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 /**
  * Abstraction of connectivity intent for traffic matching some criteria.
@@ -26,17 +25,18 @@ public abstract class ConnectivityIntent extends AbstractIntent {
 
     /**
      * Creates a connectivity intent that matches on the specified intent
-     * and applies the specified action.
+     * and applies the specified treatement.
      *
-     * @param id    intent identifier
-     * @param match traffic match
-     * @param action action
-     * @throws NullPointerException if the match or action is null
+     * @param intentId   intent identifier
+     * @param selector   traffic selector
+     * @param treatement treatement
+     * @throws NullPointerException if the selector or treatement is null
      */
-    protected ConnectivityIntent(IntentId id, TrafficSelector match, TrafficTreatment action) {
-        super(id);
-        this.selector = checkNotNull(match);
-        this.treatment = checkNotNull(action);
+    protected ConnectivityIntent(IntentId intentId, TrafficSelector selector,
+                                 TrafficTreatment treatement) {
+        super(intentId);
+        this.selector = checkNotNull(selector);
+        this.treatment = checkNotNull(treatement);
     }
 
     /**
@@ -53,7 +53,7 @@ public abstract class ConnectivityIntent extends AbstractIntent {
      *
      * @return traffic match
      */
-    public TrafficSelector getTrafficSelector() {
+    public TrafficSelector selector() {
         return selector;
     }
 
@@ -62,7 +62,7 @@ public abstract class ConnectivityIntent extends AbstractIntent {
      *
      * @return applied action
      */
-    public TrafficTreatment getTrafficTreatment() {
+    public TrafficTreatment treatment() {
         return treatment;
     }
 

+package org.onlab.onos.net.intent;
+
+import com.google.common.base.MoreObjects;
+import org.onlab.onos.net.HostId;
+import org.onlab.onos.net.flow.TrafficSelector;
+import org.onlab.onos.net.flow.TrafficTreatment;
+
+import java.util.Objects;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+/**
+ * Abstraction of end-station to end-station bidirectional connectivity.
+ */
+public class HostToHostIntent extends ConnectivityIntent {
+
+    private final HostId one;
+    private final HostId two;
+
+    /**
+     * Creates a new point-to-point intent with the supplied ingress/egress
+     * ports.
+     *
+     * @param intentId  intent identifier
+     * @param one       first host
+     * @param two       second host
+     * @param selector  action
+     * @param treatment ingress port
+     * @throws NullPointerException if {@code ingressPort} or {@code egressPort}
+     *                              is null.
+     */
+    public HostToHostIntent(IntentId intentId, HostId one, HostId two,
+                            TrafficSelector selector,
+                            TrafficTreatment treatment) {
+        super(intentId, selector, treatment);
+        this.one = checkNotNull(one);
+        this.two = checkNotNull(two);
+    }
+
+    /**
+     * Returns identifier of the first host.
+     *
+     * @return first host identifier
+     */
+    public HostId one() {
+        return one;
+    }
+
+    /**
+     * Returns identifier of the second host.
+     *
+     * @return second host identifier
+     */
+    public HostId two() {
+        return two;
+    }
+
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+        if (!super.equals(o)) {
+            return false;
+        }
+
+        HostToHostIntent that = (HostToHostIntent) o;
+        return Objects.equals(this.one, that.one)
+                && Objects.equals(this.two, that.two);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(super.hashCode(), one, two);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("id", id())
+                .add("selector", selector())
+                .add("treatment", treatment())
+                .add("one", one)
+                .add("two", two)
+                .toString();
+    }
+
+}

 package org.onlab.onos.net.intent;
 
+import org.onlab.onos.net.Link;
+
+import java.util.Collection;
+
 /**
  * Abstraction of an intent that can be installed into
  * the underlying system without additional compilation.
  */
 public interface InstallableIntent extends Intent {
+
+    /**
+     * Returns the collection of links that are required for this installable
+     * intent to exist.
+     *
+     * @return collection of links
+     */
+    // FIXME: replace this with 'NetworkResource'
+    Collection<Link> requiredLinks();
+
 }

@@ -2,7 +2,7 @@ package org.onlab.onos.net.intent;
 
 /**
  * Abstraction of an application level intent.
- *
+ * <p/>
  * Make sure that an Intent should be immutable when a new type is defined.
  */
 public interface Intent extends BatchOperationTarget {
@@ -11,5 +11,5 @@ public interface Intent extends BatchOperationTarget {
      *
      * @return intent identifier
      */
-    IntentId getId();
+    IntentId id();
 }

 package org.onlab.onos.net.intent;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.Objects;
-
-import com.google.common.base.MoreObjects;
+import org.onlab.onos.event.AbstractEvent;
 
 /**
  * A class to represent an intent related event.
  */
-public class IntentEvent {
+public class IntentEvent extends AbstractEvent<IntentEvent.Type, Intent> {
 
-    // TODO: determine a suitable parent class; if one does not exist, consider introducing one
+    public enum Type {
+        /**
+         * Signifies that a new intent has been submitted to the system.
+         */
+        SUBMITTED,
 
-    private final long time;
-    private final Intent intent;
-    private final IntentState state;
-    private final IntentState previous;
-
-    /**
-     * Creates an event describing a state change of an intent.
-     *
-     * @param intent subject intent
-     * @param state new intent state
-     * @param previous previous intent state
-     * @param time time the event created in milliseconds since start of epoch
-     * @throws NullPointerException if the intent or state is null
-     */
-    public IntentEvent(Intent intent, IntentState state, IntentState previous, long time) {
-        this.intent = checkNotNull(intent);
-        this.state = checkNotNull(state);
-        this.previous = previous;
-        this.time = time;
-    }
+        /**
+         * Signifies that an intent has been successfully installed.
+         */
+        INSTALLED,
 
-    /**
-     * Constructor for serializer.
-     */
-    protected IntentEvent() {
-        this.intent = null;
-        this.state = null;
-        this.previous = null;
-        this.time = 0;
-    }
+        /**
+         * Signifies that an intent has failed compilation or installation.
+         */
+        FAILED,
 
-    /**
-     * Returns the state of the intent which caused the event.
-     *
-     * @return the state of the intent
-     */
-    public IntentState getState() {
-        return state;
+        /**
+         * Signifies that an intent has been withdrawn from the system.
+         */
+        WITHDRAWN
     }
 
     /**
-     * Returns the previous state of the intent which caused the event.
+     * Creates an event of a given type and for the specified intent and the
+     * current time.
      *
-     * @return the previous state of the intent
-     */
-    public IntentState getPreviousState() {
-        return previous;
-    }
-
-    /**
-     * Returns the intent associated with the event.
-     *
-     * @return the intent
+     * @param type   event type
+     * @param intent subject intent
+     * @param time   time the event created in milliseconds since start of epoch
      */
-    public Intent getIntent() {
-        return intent;
+    public IntentEvent(Type type, Intent intent, long time) {
+        super(type, intent, time);
     }
 
     /**
-     * Returns the time at which the event was created.
+     * Creates an event of a given type and for the specified intent and the
+     * current time.
      *
-     * @return the time in milliseconds since start of epoch
+     * @param type   event type
+     * @param intent subject intent
      */
-    public long getTime() {
-        return time;
+    public IntentEvent(Type type, Intent intent) {
+        super(type, intent);
     }
 
-    @Override
-    public boolean equals(Object o) {
-        if (this == o) {
-            return true;
-        }
-        if (o == null || getClass() != o.getClass()) {
-            return false;
-        }
-
-        IntentEvent that = (IntentEvent) o;
-        return Objects.equals(this.intent, that.intent)
-                && Objects.equals(this.state, that.state)
-                && Objects.equals(this.previous, that.previous)
-                && Objects.equals(this.time, that.time);
-    }
-
-    @Override
-    public int hashCode() {
-        return Objects.hash(intent, state, previous, time);
-    }
-
-    @Override
-    public String toString() {
-        return MoreObjects.toStringHelper(getClass())
-                .add("intent", intent)
-                .add("state", state)
-                .add("previous", previous)
-                .add("time", time)
-                .toString();
-    }
 }

@@ -26,7 +26,7 @@ public class IntentException extends RuntimeException {
      * Constructs an exception with the specified message and the underlying cause.
      *
      * @param message the message describing the specific nature of the error
-     * @param cause the underlying cause of this error
+     * @param cause   the underlying cause of this error
      */
     public IntentException(String message, Throwable cause) {
         super(message, cause);

@@ -10,9 +10,9 @@ public interface IntentExtensionService {
     /**
      * Registers the specified compiler for the given intent class.
      *
-     * @param cls intent class
+     * @param cls      intent class
      * @param compiler intent compiler
-     * @param <T> the type of intent
+     * @param <T>      the type of intent
      */
     <T extends Intent> void registerCompiler(Class<T> cls, IntentCompiler<T> compiler);
 
@@ -34,9 +34,9 @@ public interface IntentExtensionService {
     /**
      * Registers the specified installer for the given installable intent class.
      *
-     * @param cls installable intent class
+     * @param cls       installable intent class
      * @param installer intent installer
-     * @param <T> the type of installable intent
+     * @param <T>       the type of installable intent
      */
     <T extends InstallableIntent> void registerInstaller(Class<T> cls, IntentInstaller<T> installer);
 

@@ -2,7 +2,7 @@ package org.onlab.onos.net.intent;
 
 /**
  * Intent identifier suitable as an external key.
- *
+ * <p/>
  * This class is immutable.
  */
 public final class IntentId implements BatchOperationTarget {

 package org.onlab.onos.net.intent;
 
+import org.onlab.onos.event.EventListener;
+
 /**
  * Listener for {@link IntentEvent intent events}.
  */
-public interface IntentEventListener {
-    /**
-     * Processes the specified intent event.
-     *
-     * @param event the event to process
-     */
-    void event(IntentEvent event);
+public interface IntentListener extends EventListener<IntentEvent> {
 }

 package org.onlab.onos.net.intent;
 
-import java.util.Set;
 
 /**
  * Service for application submitting or withdrawing their intents.
@@ -8,9 +7,9 @@ import java.util.Set;
 public interface IntentService {
     /**
      * Submits an intent into the system.
-     *
-     * This is an asynchronous request meaning that any compiling
-     * or installation activities may be done at later time.
+     * <p/>
+     * This is an asynchronous request meaning that any compiling or
+     * installation activities may be done at later time.
      *
      * @param intent intent to be submitted
      */
@@ -18,9 +17,9 @@ public interface IntentService {
 
     /**
      * Withdraws an intent from the system.
-     *
-     * This is an asynchronous request meaning that the environment
-     * may be affected at later time.
+     * <p/>
+     * This is an asynchronous request meaning that the environment may be
+     * affected at later time.
      *
      * @param intent intent to be withdrawn
      */
@@ -29,20 +28,27 @@ public interface IntentService {
     /**
      * Submits a batch of submit &amp; withdraw operations. Such a batch is
      * assumed to be processed together.
-     *
-     * This is an asynchronous request meaning that the environment
-     * may be affected at later time.
+     * <p/>
+     * This is an asynchronous request meaning that the environment may be
+     * affected at later time.
      *
      * @param operations batch of intent operations
      */
     void execute(IntentOperations operations);
 
     /**
-     * Returns immutable set of intents currently in the system.
+     * Returns an iterable of intents currently in the system.
      *
      * @return set of intents
      */
-    Set<Intent> getIntents();
+    Iterable<Intent> getIntents();
+
+    /**
+     * Returns the number of intents currently in the system.
+     *
+     * @return number of intents
+     */
+    long getIntentCount();
 
     /**
      * Retrieves the intent specified by its identifier.
@@ -56,7 +62,8 @@ public interface IntentService {
      * Retrieves the state of an intent by its identifier.
      *
      * @param id intent identifier
-     * @return the intent state or null if one with the given identifier is not found
+     * @return the intent state or null if one with the given identifier is not
+     * found
      */
     IntentState getIntentState(IntentId id);
 
@@ -65,12 +72,12 @@ public interface IntentService {
      *
      * @param listener listener to be added
      */
-    void addListener(IntentEventListener listener);
+    void addListener(IntentListener listener);
 
     /**
      * Removes the specified listener for intent events.
      *
      * @param listener listener to be removed
      */
-    void removeListener(IntentEventListener listener);
+    void removeListener(IntentListener listener);
 }

 package org.onlab.onos.net.intent;
 
 /**
- * This class represents the states of an intent.
- *
- * <p>
- * Note: The state is expressed as enum, but there is possibility
- * in the future that we define specific class instead of enum to improve
- * the extensibility of state definition.
- * </p>
+ * Representation of the phases an intent may attain during its lifecycle.
  */
 public enum IntentState {
-    // FIXME: requires discussion on State vs. EventType and a solid state-transition diagram
-    // TODO: consider the impact of conflict detection
-    // TODO: consider the impact that external events affect an installed intent
+
     /**
-     * The beginning state.
-     *
+     * Signifies that the intent has been submitted and will start compiling
+     * shortly. However, this compilation may not necessarily occur on the
+     * local controller instance.
+     * <p/>
      * All intent in the runtime take this state first.
      */
     SUBMITTED,
 
     /**
-     * The intent compilation has been completed.
-     *
-     * An intent translation graph (tree) is completely created.
-     * Leaves of the graph are installable intent type.
+     * Signifies that the intent is being compiled into installable intents.
+     * This is a transitional state after which the intent will enter either
+     * {@link #FAILED} state or {@link #INSTALLING} state.
+     */
+    COMPILING,
+
+    /**
+     * Signifies that the resulting installable intents are being installed
+     * into the network environment. This is a transitional state after which
+     * the intent will enter either {@link #INSTALLED} state or
+     * {@link #RECOMPILING} state.
      */
-    COMPILED,
+    INSTALLING,
 
     /**
-     * The intent has been successfully installed.
+     * The intent has been successfully installed. This is a state where the
+     * intent may remain parked until it is withdrawn by the application or
+     * until the network environment changes in some way to make the original
+     * set of installable intents untenable.
      */
     INSTALLED,
 
     /**
-     * The intent is being withdrawn.
-     *
-     * When {@link IntentService#withdraw(Intent)} is called,
-     * the intent takes this state first.
+     * Signifies that the intent is being recompiled into installable intents
+     * as an attempt to adapt to an anomaly in the network environment.
+     * This is a transitional state after which the intent will enter either
+     * {@link #FAILED} state or {@link #INSTALLING} state.
+     * <p/>
+     * Exit to the {@link #FAILED} state may be caused by failure to compile
+     * or by compiling into the same set of installable intents which have
+     * previously failed to be installed.
+     */
+    RECOMPILING,
+
+    /**
+     * Indicates that the intent is being withdrawn. This is a transitional
+     * state, triggered by invocation of the
+     * {@link IntentService#withdraw(Intent)} but one with only one outcome,
+     * which is the the intent being placed in the {@link #WITHDRAWN} state.
      */
     WITHDRAWING,
 
     /**
-     * The intent has been successfully withdrawn.
+     * Indicates that the intent has been successfully withdrawn.
      */
     WITHDRAWN,
 
     /**
-     * The intent has failed to be compiled, installed, or withdrawn.
-     *
-     * When the intent failed to be withdrawn, it is still, at least partially installed.
+     * Signifies that the intent has failed compiling, installing or
+     * recompiling states.
      */
-    FAILED,
+    FAILED
 }

+package org.onlab.onos.net.intent;
+
+import org.onlab.onos.store.Store;
+
+import java.util.List;
+
+/**
+ * Manages inventory of end-station intents; not intended for direct use.
+ */
+public interface IntentStore extends Store<IntentEvent, IntentStoreDelegate> {
+
+    /**
+     * Submits a new intent into the store. If the returned event is not
+     * null, the manager is expected to dispatch the event and then to kick
+     * off intent compilation process. Otherwise, another node has been elected
+     * to perform the compilation process and the node will learn about
+     * the submittal and results of the intent compilation via the delegate
+     * mechanism.
+     *
+     * @param intent intent to be submitted
+     * @return event indicating the intent was submitted or null if no
+     * change resulted, e.g. duplicate intent
+     */
+    IntentEvent createIntent(Intent intent);
+
+    /**
+     * Removes the specified intent from the inventory.
+     *
+     * @param intentId intent identification
+     * @return removed state transition event or null if intent was not found
+     */
+    IntentEvent removeIntent(IntentId intentId);
+
+    /**
+     * Returns the number of intents in the store.
+     */
+    long getIntentCount();
+
+    /**
+     * Returns a collection of all intents in the store.
+     *
+     * @return iterable collection of all intents
+     */
+    Iterable<Intent> getIntents();
+
+    /**
+     * Returns the intent with the specified identifer.
+     *
+     * @param intentId intent identification
+     * @return intent or null if not found
+     */
+    Intent getIntent(IntentId intentId);
+
+    /**
+     * Returns the state of the specified intent.
+     *
+     * @param intentId intent identification
+     * @return current intent state
+     */
+    IntentState getIntentState(IntentId intentId);
+
+    /**
+     * Sets the state of the specified intent to the new state.
+     *
+     * @param intent   intent whose state is to be changed
+     * @param newState new state
+     * @return state transition event
+     */
+    IntentEvent setState(Intent intent, IntentState newState);
+
+    /**
+     * Adds the installable intents which resulted from compilation of the
+     * specified original intent.
+     *
+     * @param intentId           original intent identifier
+     * @param installableIntents compiled installable intents
+     */
+    void addInstallableIntents(IntentId intentId,
+                               List<InstallableIntent> installableIntents);
+
+    /**
+     * Returns the list of the installable events associated with the specified
+     * original intent.
+     *
+     * @param intentId original intent identifier
+     * @return compiled installable intents
+     */
+    List<InstallableIntent> getInstallableIntents(IntentId intentId);
+
+    // TODO: this should be triggered from with the store as a result of removeIntent call
+
+    /**
+     * Removes any installable intents which resulted from compilation of the
+     * specified original intent.
+     *
+     * @param intentId original intent identifier
+     * @return compiled state transition event
+     */
+    void removeInstalledIntents(IntentId intentId);
+
+}

+package org.onlab.onos.net.intent;
+
+import org.onlab.onos.store.StoreDelegate;
+
+/**
+ * Intent store delegate abstraction.
+ */
+public interface IntentStoreDelegate extends StoreDelegate<IntentEvent> {
+}

 package org.onlab.onos.net.intent;
 
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.Objects;
-import java.util.Set;
-
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Sets;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.flow.TrafficSelector;
 import org.onlab.onos.net.flow.TrafficTreatment;
 
-import com.google.common.base.MoreObjects;
-import com.google.common.collect.Sets;
+import java.util.Objects;
+import java.util.Set;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 /**
  * Abstraction of multiple source to single destination connectivity intent.
  */
 public class MultiPointToSinglePointIntent extends ConnectivityIntent {
 
-    private final Set<ConnectPoint> ingressPorts;
-    private final ConnectPoint egressPort;
+    private final Set<ConnectPoint> ingressPoints;
+    private final ConnectPoint egressPoint;
 
     /**
      * Creates a new multi-to-single point connectivity intent for the specified
@@ -28,23 +27,25 @@ public class MultiPointToSinglePointIntent extends ConnectivityIntent {
      * @param id           intent identifier
      * @param match        traffic match
      * @param action       action
-     * @param ingressPorts set of ports from which ingress traffic originates
-     * @param egressPort   port to which traffic will egress
-     * @throws NullPointerException if {@code ingressPorts} or
-     * {@code egressPort} is null.
-     * @throws IllegalArgumentException if the size of {@code ingressPorts} is
-     * not more than 1
+     * @param ingressPoints set of ports from which ingress traffic originates
+     * @param egressPoint   port to which traffic will egress
+     * @throws NullPointerException     if {@code ingressPoints} or
+     *                                  {@code egressPoint} is null.
+     * @throws IllegalArgumentException if the size of {@code ingressPoints} is
+     *                                  not more than 1
      */
-    public MultiPointToSinglePointIntent(IntentId id, TrafficSelector match, TrafficTreatment action,
-            Set<ConnectPoint> ingressPorts, ConnectPoint egressPort) {
+    public MultiPointToSinglePointIntent(IntentId id, TrafficSelector match,
+                                         TrafficTreatment action,
+                                         Set<ConnectPoint> ingressPoints,
+                                         ConnectPoint egressPoint) {
         super(id, match, action);
 
-        checkNotNull(ingressPorts);
-        checkArgument(!ingressPorts.isEmpty(),
-                "there should be at least one ingress port");
+        checkNotNull(ingressPoints);
+        checkArgument(!ingressPoints.isEmpty(),
+                      "there should be at least one ingress port");
 
-        this.ingressPorts = Sets.newHashSet(ingressPorts);
-        this.egressPort = checkNotNull(egressPort);
+        this.ingressPoints = Sets.newHashSet(ingressPoints);
+        this.egressPoint = checkNotNull(egressPoint);
     }
 
     /**
@@ -52,8 +53,8 @@ public class MultiPointToSinglePointIntent extends ConnectivityIntent {
      */
     protected MultiPointToSinglePointIntent() {
         super();
-        this.ingressPorts = null;
-        this.egressPort = null;
+        this.ingressPoints = null;
+        this.egressPoint = null;
     }
 
     /**
@@ -62,8 +63,8 @@ public class MultiPointToSinglePointIntent extends ConnectivityIntent {
      *
      * @return set of ingress ports
      */
-    public Set<ConnectPoint> getIngressPorts() {
-        return ingressPorts;
+    public Set<ConnectPoint> ingressPoints() {
+        return ingressPoints;
     }
 
     /**
@@ -71,8 +72,8 @@ public class MultiPointToSinglePointIntent extends ConnectivityIntent {
      *
      * @return egress port
      */
-    public ConnectPoint getEgressPort() {
-        return egressPort;
+    public ConnectPoint egressPoint() {
+        return egressPoint;
     }
 
     @Override
@@ -88,23 +89,23 @@ public class MultiPointToSinglePointIntent extends ConnectivityIntent {
         }
 
         MultiPointToSinglePointIntent that = (MultiPointToSinglePointIntent) o;
-        return Objects.equals(this.ingressPorts, that.ingressPorts)
-                && Objects.equals(this.egressPort, that.egressPort);
+        return Objects.equals(this.ingressPoints, that.ingressPoints)
+                && Objects.equals(this.egressPoint, that.egressPoint);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), ingressPorts, egressPort);
+        return Objects.hash(super.hashCode(), ingressPoints, egressPoint);
     }
 
     @Override
     public String toString() {
         return MoreObjects.toStringHelper(getClass())
-                .add("id", getId())
-                .add("match", getTrafficSelector())
-                .add("action", getTrafficTreatment())
-                .add("ingressPorts", getIngressPorts())
-                .add("egressPort", getEgressPort())
+                .add("id", id())
+                .add("match", selector())
+                .add("action", treatment())
+                .add("ingressPoints", ingressPoints())
+                .add("egressPoint", egressPoint())
                 .toString();
     }
 }

@@ -3,30 +3,30 @@ package org.onlab.onos.net.intent;
 import org.onlab.onos.net.ConnectPoint;
 
 // TODO: consider if this intent should be sub-class of ConnectivityIntent
+
 /**
  * An optical layer Intent for a connectivity from a transponder port to another
  * transponder port.
- * <p>
+ * <p/>
  * This class doesn't accepts lambda specifier. This class computes path between
  * ports and assign lambda automatically. The lambda can be specified using
  * OpticalPathFlow class.
  */
 public class OpticalConnectivityIntent extends AbstractIntent {
-    protected ConnectPoint srcConnectPoint;
-    protected ConnectPoint dstConnectPoint;
+    protected ConnectPoint src;
+    protected ConnectPoint dst;
 
     /**
      * Constructor.
      *
-     * @param id ID for this new Intent object.
-     * @param srcConnectPoint The source transponder port.
-     * @param dstConnectPoint The destination transponder port.
+     * @param id  ID for this new Intent object.
+     * @param src The source transponder port.
+     * @param dst The destination transponder port.
      */
-    public OpticalConnectivityIntent(IntentId id,
-            ConnectPoint srcConnectPoint, ConnectPoint dstConnectPoint) {
+    public OpticalConnectivityIntent(IntentId id, ConnectPoint src, ConnectPoint dst) {
         super(id);
-        this.srcConnectPoint = srcConnectPoint;
-        this.dstConnectPoint = dstConnectPoint;
+        this.src = src;
+        this.dst = dst;
     }
 
     /**
@@ -34,8 +34,8 @@ public class OpticalConnectivityIntent extends AbstractIntent {
      */
     protected OpticalConnectivityIntent() {
         super();
-        this.srcConnectPoint = null;
-        this.dstConnectPoint = null;
+        this.src = null;
+        this.dst = null;
     }
 
     /**
@@ -44,7 +44,7 @@ public class OpticalConnectivityIntent extends AbstractIntent {
      * @return The source transponder port.
      */
     public ConnectPoint getSrcConnectPoint() {
-        return srcConnectPoint;
+        return src;
     }
 
     /**
@@ -52,7 +52,7 @@ public class OpticalConnectivityIntent extends AbstractIntent {
      *
      * @return The source transponder port.
      */
-    public ConnectPoint getDstConnectPoint() {
-        return dstConnectPoint;
+    public ConnectPoint getDst() {
+        return dst;
     }
 }

 package org.onlab.onos.net.intent;
 
-import java.util.Objects;
-
+import com.google.common.base.MoreObjects;
 import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.Link;
 import org.onlab.onos.net.Path;
 import org.onlab.onos.net.flow.TrafficSelector;
 import org.onlab.onos.net.flow.TrafficTreatment;
 
-import com.google.common.base.MoreObjects;
+import java.util.Collection;
+import java.util.Objects;
 
 /**
  * Abstraction of explicitly path specified connectivity intent.
  */
-public class PathIntent extends PointToPointIntent {
+public class PathIntent extends PointToPointIntent implements InstallableIntent {
 
     private final Path path;
 
@@ -45,7 +46,7 @@ public class PathIntent extends PointToPointIntent {
      *
      * @return traversed links
      */
-    public Path getPath() {
+    public Path path() {
         return path;
     }
 
@@ -78,12 +79,18 @@ public class PathIntent extends PointToPointIntent {
     @Override
     public String toString() {
         return MoreObjects.toStringHelper(getClass())
-                .add("id", getId())
-                .add("match", getTrafficSelector())
-                .add("action", getTrafficTreatment())
-                .add("ingressPort", getIngressPort())
-                .add("egressPort", getEgressPort())
+                .add("id", id())
+                .add("match", selector())
+                .add("action", treatment())
+                .add("ingressPort", ingressPoint())
+                .add("egressPort", egressPoint())
                 .add("path", path)
                 .toString();
     }
+
+    @Override
+    public Collection<Link> requiredLinks() {
+        return path.links();
+    }
+
 }

 package org.onlab.onos.net.intent;
 
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.Objects;
-
+import com.google.common.base.MoreObjects;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.flow.TrafficSelector;
 import org.onlab.onos.net.flow.TrafficTreatment;
 
-import com.google.common.base.MoreObjects;
+import java.util.Objects;
+
+import static com.google.common.base.Preconditions.checkNotNull;
 
 /**
  * Abstraction of point-to-point connectivity.
  */
 public class PointToPointIntent extends ConnectivityIntent {
 
-    private final ConnectPoint ingressPort;
-    private final ConnectPoint egressPort;
+    private final ConnectPoint ingressPoint;
+    private final ConnectPoint egressPoint;
 
     /**
      * Creates a new point-to-point intent with the supplied ingress/egress
      * ports.
      *
-     * @param id          intent identifier
-     * @param match       traffic match
-     * @param action      action
-     * @param ingressPort ingress port
-     * @param egressPort  egress port
-     * @throws NullPointerException if {@code ingressPort} or {@code egressPort} is null.
+     * @param id           intent identifier
+     * @param selector     traffic selector
+     * @param treatment    treatment
+     * @param ingressPoint ingress port
+     * @param egressPoint  egress port
+     * @throws NullPointerException if {@code ingressPoint} or {@code egressPoints} is null.
      */
-    public PointToPointIntent(IntentId id, TrafficSelector match, TrafficTreatment action,
-                              ConnectPoint ingressPort, ConnectPoint egressPort) {
-        super(id, match, action);
-        this.ingressPort = checkNotNull(ingressPort);
-        this.egressPort = checkNotNull(egressPort);
+    public PointToPointIntent(IntentId id, TrafficSelector selector,
+                              TrafficTreatment treatment,
+                              ConnectPoint ingressPoint,
+                              ConnectPoint egressPoint) {
+        super(id, selector, treatment);
+        this.ingressPoint = checkNotNull(ingressPoint);
+        this.egressPoint = checkNotNull(egressPoint);
     }
 
     /**
@@ -41,8 +42,8 @@ public class PointToPointIntent extends ConnectivityIntent {
      */
     protected PointToPointIntent() {
         super();
-        this.ingressPort = null;
-        this.egressPort = null;
+        this.ingressPoint = null;
+        this.egressPoint = null;
     }
 
     /**
@@ -51,8 +52,8 @@ public class PointToPointIntent extends ConnectivityIntent {
      *
      * @return ingress port
      */
-    public ConnectPoint getIngressPort() {
-        return ingressPort;
+    public ConnectPoint ingressPoint() {
+        return ingressPoint;
     }
 
     /**
@@ -60,8 +61,8 @@ public class PointToPointIntent extends ConnectivityIntent {
      *
      * @return egress port
      */
-    public ConnectPoint getEgressPort() {
-        return egressPort;
+    public ConnectPoint egressPoint() {
+        return egressPoint;
     }
 
     @Override
@@ -77,23 +78,23 @@ public class PointToPointIntent extends ConnectivityIntent {
         }
 
         PointToPointIntent that = (PointToPointIntent) o;
-        return Objects.equals(this.ingressPort, that.ingressPort)
-                && Objects.equals(this.egressPort, that.egressPort);
+        return Objects.equals(this.ingressPoint, that.ingressPoint)
+                && Objects.equals(this.egressPoint, that.egressPoint);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), ingressPort, egressPort);
+        return Objects.hash(super.hashCode(), ingressPoint, egressPoint);
     }
 
     @Override
     public String toString() {
         return MoreObjects.toStringHelper(getClass())
-                .add("id", getId())
-                .add("match", getTrafficSelector())
-                .add("action", getTrafficTreatment())
-                .add("ingressPort", ingressPort)
-                .add("egressPort", egressPort)
+                .add("id", id())
+                .add("match", selector())
+                .add("action", treatment())
+                .add("ingressPoint", ingressPoint)
+                .add("egressPoints", egressPoint)
                 .toString();
     }
 

 package org.onlab.onos.net.intent;
 
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkNotNull;
-
-import java.util.Objects;
-import java.util.Set;
-
+import com.google.common.base.MoreObjects;
+import com.google.common.collect.Sets;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.flow.TrafficSelector;
 import org.onlab.onos.net.flow.TrafficTreatment;
 
-import com.google.common.base.MoreObjects;
-import com.google.common.collect.Sets;
+import java.util.Objects;
+import java.util.Set;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 /**
  * Abstraction of single source, multiple destination connectivity intent.
  */
 public class SinglePointToMultiPointIntent extends ConnectivityIntent {
 
-    private final ConnectPoint ingressPort;
-    private final Set<ConnectPoint> egressPorts;
+    private final ConnectPoint ingressPoint;
+    private final Set<ConnectPoint> egressPoints;
 
     /**
      * Creates a new single-to-multi point connectivity intent.
      *
-     * @param id          intent identifier
-     * @param match       traffic match
-     * @param action      action
-     * @param ingressPort port on which traffic will ingress
-     * @param egressPorts set of ports on which traffic will egress
-     * @throws NullPointerException if {@code ingressPort} or
-     * {@code egressPorts} is null
-     * @throws IllegalArgumentException if the size of {@code egressPorts} is
-     * not more than 1
+     * @param id           intent identifier
+     * @param selector     traffic selector
+     * @param treatment    treatment
+     * @param ingressPoint port on which traffic will ingress
+     * @param egressPoints set of ports on which traffic will egress
+     * @throws NullPointerException     if {@code ingressPoint} or
+     *                                  {@code egressPoints} is null
+     * @throws IllegalArgumentException if the size of {@code egressPoints} is
+     *                                  not more than 1
      */
-    public SinglePointToMultiPointIntent(IntentId id, TrafficSelector match, TrafficTreatment action,
-                                         ConnectPoint ingressPort,
-                                         Set<ConnectPoint> egressPorts) {
-        super(id, match, action);
-
-        checkNotNull(egressPorts);
-        checkArgument(!egressPorts.isEmpty(),
-                "there should be at least one egress port");
-
-        this.ingressPort = checkNotNull(ingressPort);
-        this.egressPorts = Sets.newHashSet(egressPorts);
+    public SinglePointToMultiPointIntent(IntentId id, TrafficSelector selector,
+                                         TrafficTreatment treatment,
+                                         ConnectPoint ingressPoint,
+                                         Set<ConnectPoint> egressPoints) {
+        super(id, selector, treatment);
+
+        checkNotNull(egressPoints);
+        checkArgument(!egressPoints.isEmpty(),
+                      "there should be at least one egress port");
+
+        this.ingressPoint = checkNotNull(ingressPoint);
+        this.egressPoints = Sets.newHashSet(egressPoints);
     }
 
     /**
@@ -52,8 +52,8 @@ public class SinglePointToMultiPointIntent extends ConnectivityIntent {
      */
     protected SinglePointToMultiPointIntent() {
         super();
-        this.ingressPort = null;
-        this.egressPorts = null;
+        this.ingressPoint = null;
+        this.egressPoints = null;
     }
 
     /**
@@ -61,8 +61,8 @@ public class SinglePointToMultiPointIntent extends ConnectivityIntent {
      *
      * @return ingress port
      */
-    public ConnectPoint getIngressPort() {
-        return ingressPort;
+    public ConnectPoint ingressPoint() {
+        return ingressPoint;
     }
 
     /**
@@ -70,8 +70,8 @@ public class SinglePointToMultiPointIntent extends ConnectivityIntent {
      *
      * @return set of egress ports
      */
-    public Set<ConnectPoint> getEgressPorts() {
-        return egressPorts;
+    public Set<ConnectPoint> egressPoints() {
+        return egressPoints;
     }
 
     @Override
@@ -87,23 +87,23 @@ public class SinglePointToMultiPointIntent extends ConnectivityIntent {
         }
 
         SinglePointToMultiPointIntent that = (SinglePointToMultiPointIntent) o;
-        return Objects.equals(this.ingressPort, that.ingressPort)
-                && Objects.equals(this.egressPorts, that.egressPorts);
+        return Objects.equals(this.ingressPoint, that.ingressPoint)
+                && Objects.equals(this.egressPoints, that.egressPoints);
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), ingressPort, egressPorts);
+        return Objects.hash(super.hashCode(), ingressPoint, egressPoints);
     }
 
     @Override
     public String toString() {
         return MoreObjects.toStringHelper(getClass())
-                .add("id", getId())
-                .add("match", getTrafficSelector())
-                .add("action", getTrafficTreatment())
-                .add("ingressPort", ingressPort)
-                .add("egressPort", egressPorts)
+                .add("id", id())
+                .add("match", selector())
+                .add("action", treatment())
+                .add("ingressPoint", ingressPoint)
+                .add("egressPort", egressPoints)
                 .toString();
     }
 

 /**
- * Intent Package. TODO
+ * Set of abstractions for conveying high-level intents for treatment of
+ * selected network traffic by allowing applications to express the
+ * <em>what</em> rather than the <em>how</em>. This makes such instructions
+ * largely independent of topology and device specifics, thus allowing them to
+ * survive topology mutations.
+ * <p/>
+ * The controller core provides a suite of built-in intents and their compilers
+ * and installers. However, the intent framework is extensible in that it allows
+ * additional intents and their compilers or installers to be added
+ * dynamically at run-time. This allows others to enhance the initial arsenal of
+ * connectivity and policy-based intents available in base controller software.
+ * <p/>
+ * The following diagram depicts the state transition diagram for each top-level intent:<br>
+ * <img src="doc-files/intent-states.png" alt="ONOS intent states">
+ * <p/>
+ * The controller core accepts the intent specifications and translates them, via a
+ * process referred to as intent compilation, to installable intents, which are
+ * essentially actionable operations on the network environment.
+ * These actions are carried out by intent installation process, which results
+ * in some changes to the environment, e.g. tunnel links being provisioned,
+ * flow rules being installed on the data-plane, optical lambdas being reserved.
+ * <p/>
+ * After an intent is submitted by an application, it will be sent immediately
+ * (but asynchronously) into a compiling phase, then to installing phase and if
+ * all goes according to plan into installed state. Once an application decides
+ * it no longer wishes the intent to hold, it can withdraw it. This describes
+ * the nominal flow. However, it may happen that some issue is encountered.
+ * For example, an application may ask for an objective that is not currently
+ * achievable, e.g. connectivity across to unconnected network segments.
+ * If this is the case, the compiling phase may fail to produce a set of
+ * installable intents and instead result in a failed compile. If this occurs,
+ * only a change in the environment can trigger a transition back to the
+ * compiling state.
+ * <p/>
+ * Similarly, an issue may be encountered during the installation phase in
+ * which case the framework will attempt to recompile the intent to see if an
+ * alternate approach is available. If so, the intent will be sent back to
+ * installing phase. Otherwise, it will be parked in the failed state. Another
+ * scenario that’s very likely to be encountered is where the intent is
+ * successfully compiled and installed, but due to some topology event, such
+ * as a downed or downgraded link, loss of throughput may occur or connectivity
+ * may be lost altogether, thus impacting the viability of a previously
+ * satisfied intent. If this occurs, the framework will attempt to recompile
+ * the intent, and if an alternate approach is available, its installation
+ * will be attempted. Otherwise, the original top-level intent will be parked
+ * in the failed state.
+ * <p/>
+ * Please note that all *ing states, depicted in orange, are transitional and
+ * are expected to last only a brief amount of time. The rest of the states
+ * are parking states where the intent may spent some time; except for the
+ * submitted state of course. There, the intent may pause, but only briefly,
+ * while the system determines where to perform the compilation or while it
+ * performs global recomputation/optimization across all prior intents.
  */
-
 package org.onlab.onos.net.intent;
\ No newline at end of file

 package org.onlab.onos.net.link;
 
 import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.Description;
 import org.onlab.onos.net.Link;
 
 /**
  * Describes an infrastructure link.
  */
-public interface LinkDescription {
+public interface LinkDescription extends Description {
 
     /**
      * Returns the link source.

@@ -24,7 +24,7 @@ public abstract class DefaultPacketContext implements PacketContext {
         this.inPkt = inPkt;
         this.outPkt = outPkt;
         this.block = new AtomicBoolean(block);
-        this.builder = new DefaultTrafficTreatment.Builder();
+        this.builder = DefaultTrafficTreatment.builder();
     }
 
     @Override

@@ -3,6 +3,7 @@ package org.onlab.onos.net.provider;
 import java.util.Objects;
 
 import static com.google.common.base.MoreObjects.toStringHelper;
+import static com.google.common.base.Preconditions.checkNotNull;
 
 /**
  * External identity of a {@link org.onlab.onos.net.provider.Provider} family.
@@ -19,10 +20,22 @@ import static com.google.common.base.MoreObjects.toStringHelper;
  */
 public class ProviderId {
 
+    /**
+     * Represents no provider ID.
+     */
+    public static final ProviderId NONE = new ProviderId();
+
     private final String scheme;
     private final String id;
     private final boolean ancillary;
 
+    // For serialization
+    private ProviderId() {
+        scheme = null;
+        id = null;
+        ancillary = false;
+    }
+
     /**
      * Creates a new primary provider identifier from the specified string.
      * The providers are expected to follow the reverse DNS convention, e.g.
@@ -45,8 +58,8 @@ public class ProviderId {
      * @param ancillary ancillary provider indicator
      */
     public ProviderId(String scheme, String id, boolean ancillary) {
-        this.scheme = scheme;
-        this.id = id;
+        this.scheme = checkNotNull(scheme, "Scheme cannot be null");
+        this.id = checkNotNull(id, "ID cannot be null");
         this.ancillary = ancillary;
     }
 

 package org.onlab.onos.net.proxyarp;
 
+import org.onlab.onos.net.packet.PacketContext;
 import org.onlab.packet.Ethernet;
 import org.onlab.packet.IpPrefix;
 
@@ -33,4 +34,12 @@ public interface ProxyArpService {
      */
     void forward(Ethernet eth);
 
+    /**
+     * Handles a arp packet.
+     * Replies to arp requests and forwards request to the  right place.
+     * @param context the packet context to handle
+     * @return true if handled, false otherwise.
+     */
+    boolean handleArp(PacketContext context);
+
 }

 package org.onlab.onos.net.topology;
 
 import org.onlab.onos.event.AbstractEvent;
+import org.onlab.onos.event.Event;
+
+import java.util.List;
 
 /**
  * Describes network topology event.
  */
 public class TopologyEvent extends AbstractEvent<TopologyEvent.Type, Topology> {
 
+    private final List<Event> reasons;
+
     /**
      * Type of topology events.
      */
@@ -23,9 +28,11 @@ public class TopologyEvent extends AbstractEvent<TopologyEvent.Type, Topology> {
      *
      * @param type     topology event type
      * @param topology event topology subject
+     * @param reasons  list of events that triggered topology change
      */
-    public TopologyEvent(Type type, Topology topology) {
+    public TopologyEvent(Type type, Topology topology, List<Event> reasons) {
         super(type, topology);
+        this.reasons = reasons;
     }
 
     /**
@@ -33,10 +40,24 @@ public class TopologyEvent extends AbstractEvent<TopologyEvent.Type, Topology> {
      *
      * @param type     link event type
      * @param topology event topology subject
+     * @param reasons  list of events that triggered topology change
      * @param time     occurrence time
      */
-    public TopologyEvent(Type type, Topology topology, long time) {
+    public TopologyEvent(Type type, Topology topology, List<Event> reasons,
+                         long time) {
         super(type, topology, time);
+        this.reasons = reasons;
+    }
+
+
+    /**
+     * Returns the list of events that triggered the topology change.
+     *
+     * @return list of events responsible for change in topology; null if
+     * initial topology computation
+     */
+    public List<Event> reasons() {
+        return reasons;
     }
 
 }

+package org.onlab.onos.store;
+
+import org.onlab.onos.cluster.MastershipTerm;
+import org.onlab.onos.net.DeviceId;
+
+//TODO: Consider renaming to DeviceClockProviderService?
+/**
+* Interface for feeding term information to a logical clock service
+* that vends per device timestamps.
+*/
+public interface ClockProviderService {
+
+    /**
+     * Updates the mastership term for the specified deviceId.
+     *
+     * @param deviceId device identifier.
+     * @param term mastership term.
+     */
+    public void setMastershipTerm(DeviceId deviceId, MastershipTerm term);
+}

 package org.onlab.onos.store;
 
-import org.onlab.onos.cluster.MastershipTerm;
 import org.onlab.onos.net.DeviceId;
 
 // TODO: Consider renaming to DeviceClockService?
@@ -15,12 +14,4 @@ public interface ClockService {
      * @return timestamp.
      */
     public Timestamp getTimestamp(DeviceId deviceId);
-
-    // TODO: Should this be here or separate as Admin service, etc.?
-    /**
-     * Updates the mastership term for the specified deviceId.
-     * @param deviceId device identifier.
-     * @param term mastership term.
-     */
-    public void setMastershipTerm(DeviceId deviceId, MastershipTerm term);
 }

@@ -2,7 +2,15 @@ package org.onlab.onos.store;
 
 /**
  * Opaque version structure.
+ * <p>
+ * Classes implementing this interface must also implement
+ * {@link #hashCode()} and {@link #equals(Object)}.
  */
 public interface Timestamp extends Comparable<Timestamp> {
 
+    @Override
+    public abstract int hashCode();
+
+    @Override
+    public abstract boolean equals(Object obj);
 }

+package org.onlab.onos;
+
+import com.google.common.testing.EqualsTester;
+import org.junit.Test;
+
+import static org.junit.Assert.*;
+import static org.onlab.onos.Version.version;
+
+/**
+ * Tests of the version descriptor.
+ */
+public class VersionTest {
+
+    @Test
+    public void fromParts() {
+        Version v = version(1, 2, 3, "4321");
+        assertEquals("wrong major", 1, v.major());
+        assertEquals("wrong minor", 2, v.minor());
+        assertEquals("wrong patch", 3, v.patch());
+        assertEquals("wrong build", "4321", v.build());
+    }
+
+    @Test
+    public void fromString() {
+        Version v = version("1.2.3.4321");
+        assertEquals("wrong major", 1, v.major());
+        assertEquals("wrong minor", 2, v.minor());
+        assertEquals("wrong patch", 3, v.patch());
+        assertEquals("wrong build", "4321", v.build());
+    }
+
+    @Test
+    public void snapshot() {
+        Version v = version("1.2.3-SNAPSHOT");
+        assertEquals("wrong major", 1, v.major());
+        assertEquals("wrong minor", 2, v.minor());
+        assertEquals("wrong patch", 3, v.patch());
+        assertEquals("wrong build", "SNAPSHOT", v.build());
+    }
+
+    @Test
+    public void testEquals() {
+        new EqualsTester()
+                .addEqualityGroup(version("1.2.3.4321"), version(1, 2, 3, "4321"))
+                .addEqualityGroup(version("1.9.3.4321"), version(1, 9, 3, "4321"))
+                .addEqualityGroup(version("1.2.8.4321"), version(1, 2, 8, "4321"))
+                .addEqualityGroup(version("1.2.3.x"), version(1, 2, 3, "x"))
+                .testEquals();
+    }
+}
\ No newline at end of file

@@ -33,4 +33,4 @@ public class ConnectPointTest {
                 .addEqualityGroup(new ConnectPoint(DID2, P1), new ConnectPoint(DID2, P1))
                 .testEquals();
     }
-}
\ No newline at end of file
+}

@@ -36,6 +36,23 @@ public class DefaultAnnotationsTest {
     }
 
     @Test
+    public void union() {
+        annotations = builder().set("foo", "1").set("bar", "2").remove("buz").build();
+        assertEquals("incorrect keys", of("foo", "bar", "buz"), annotations.keys());
+
+        SparseAnnotations updates = builder().remove("foo").set("bar", "3").set("goo", "4").remove("fuzz").build();
+
+        SparseAnnotations result = DefaultAnnotations.union(annotations, updates);
+
+        assertTrue("remove instruction in original remains", result.isRemoved("buz"));
+        assertTrue("remove instruction in update remains", result.isRemoved("fuzz"));
+        assertEquals("incorrect keys", of("buz", "goo", "bar", "fuzz"), result.keys());
+        assertNull("incorrect value", result.value("foo"));
+        assertEquals("incorrect value", "3", result.value("bar"));
+        assertEquals("incorrect value", "4", result.value("goo"));
+    }
+
+    @Test
     public void merge() {
         annotations = builder().set("foo", "1").set("bar", "2").build();
         assertEquals("incorrect keys", of("foo", "bar"), annotations.keys());
@@ -65,4 +82,4 @@ public class DefaultAnnotationsTest {
         DefaultAnnotations.merge(null, null);
     }
 
-}
\ No newline at end of file
+}

@@ -58,7 +58,5 @@ public class DefaultDeviceTest {
         assertEquals("incorrect hw", HW, device.hwVersion());
         assertEquals("incorrect sw", SW, device.swVersion());
         assertEquals("incorrect serial", SN1, device.serialNumber());
-        assertEquals("incorrect serial", SN1, device.serialNumber());
     }
-
-}
\ No newline at end of file
+}

@@ -5,6 +5,7 @@ import org.junit.Test;
 import org.onlab.onos.net.provider.ProviderId;
 
 import static org.junit.Assert.assertEquals;
+import static org.onlab.onos.net.DefaultEdgeLink.createEdgeLink;
 import static org.onlab.onos.net.DefaultLinkTest.cp;
 import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.onlab.onos.net.HostId.hostId;
@@ -55,4 +56,24 @@ public class DefaultEdgeLinkTest {
         assertEquals("incorrect time", 123L, link.hostLocation().time());
     }
 
+    @Test
+    public void phantomIngress() {
+        HostLocation hostLocation = new HostLocation(DID1, P1, 123L);
+        EdgeLink link = createEdgeLink(hostLocation, true);
+        assertEquals("incorrect dst", hostLocation, link.dst());
+        assertEquals("incorrect type", Link.Type.EDGE, link.type());
+        assertEquals("incorrect connect point", hostLocation, link.hostLocation());
+        assertEquals("incorrect time", 123L, link.hostLocation().time());
+    }
+
+    @Test
+    public void phantomEgress() {
+        ConnectPoint hostLocation = new ConnectPoint(DID1, P1);
+        EdgeLink link = createEdgeLink(hostLocation, false);
+        assertEquals("incorrect src", hostLocation, link.src());
+        assertEquals("incorrect type", Link.Type.EDGE, link.type());
+        assertEquals("incorrect connect point", hostLocation, link.hostLocation());
+        assertEquals("incorrect time", 0L, link.hostLocation().time());
+    }
+
 }

@@ -39,7 +39,7 @@ public class DeviceEventTest extends AbstractEventTest {
         Device device = createDevice();
         Port port = new DefaultPort(device, PortNumber.portNumber(123), true);
         long before = System.currentTimeMillis();
-        DeviceEvent event = new DeviceEvent(DeviceEvent.Type.DEVICE_ADDED, device);
+        DeviceEvent event = new DeviceEvent(DeviceEvent.Type.DEVICE_ADDED, device, port);
         long after = System.currentTimeMillis();
         validateEvent(event, DeviceEvent.Type.DEVICE_ADDED, device, before, after);
     }

@@ -16,8 +16,8 @@ import org.onlab.onos.net.flow.TrafficTreatment;
 public abstract class ConnectivityIntentTest extends IntentTest {
 
     public static final IntentId IID = new IntentId(123);
-    public static final TrafficSelector MATCH = (new DefaultTrafficSelector.Builder()).build();
-    public static final TrafficTreatment NOP = (new DefaultTrafficTreatment.Builder()).build();
+    public static final TrafficSelector MATCH = DefaultTrafficSelector.builder().build();
+    public static final TrafficTreatment NOP = DefaultTrafficTreatment.builder().build();
 
     public static final ConnectPoint P1 = new ConnectPoint(DeviceId.deviceId("111"), PortNumber.portNumber(0x1));
     public static final ConnectPoint P2 = new ConnectPoint(DeviceId.deviceId("222"), PortNumber.portNumber(0x2));

@@ -11,19 +11,19 @@ import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 
 /**
- * Fake implementation of the intent service to assist in developing tests
- * of the interface contract.
+ * Fake implementation of the intent service to assist in developing tests of
+ * the interface contract.
  */
 public class FakeIntentManager implements TestableIntentService {
 
     private final Map<IntentId, Intent> intents = new HashMap<>();
     private final Map<IntentId, IntentState> intentStates = new HashMap<>();
     private final Map<IntentId, List<InstallableIntent>> installables = new HashMap<>();
-    private final Set<IntentEventListener> listeners = new HashSet<>();
+    private final Set<IntentListener> listeners = new HashSet<>();
 
     private final Map<Class<? extends Intent>, IntentCompiler<? extends Intent>> compilers = new HashMap<>();
-    private final Map<Class<? extends InstallableIntent>,
-            IntentInstaller<? extends InstallableIntent>> installers = new HashMap<>();
+    private final Map<Class<? extends InstallableIntent>, IntentInstaller<? extends InstallableIntent>> installers
+        = new HashMap<>();
 
     private final ExecutorService executor = Executors.newSingleThreadExecutor();
     private final List<IntentException> exceptions = new ArrayList<>();
@@ -40,8 +40,7 @@ public class FakeIntentManager implements TestableIntentService {
             @Override
             public void run() {
                 try {
-                    List<InstallableIntent> installable = compileIntent(intent);
-                    installIntents(intent, installable);
+                    executeCompilingPhase(intent);
                 } catch (IntentException e) {
                     exceptions.add(e);
                 }
@@ -55,8 +54,8 @@ public class FakeIntentManager implements TestableIntentService {
             @Override
             public void run() {
                 try {
-                    List<InstallableIntent> installable = getInstallable(intent.getId());
-                    uninstallIntents(intent, installable);
+                    List<InstallableIntent> installable = getInstallable(intent.id());
+                    executeWithdrawingPhase(intent, installable);
                 } catch (IntentException e) {
                     exceptions.add(e);
                 }
@@ -76,61 +75,68 @@ public class FakeIntentManager implements TestableIntentService {
 
     private <T extends InstallableIntent> IntentInstaller<T> getInstaller(T intent) {
         @SuppressWarnings("unchecked")
-        IntentInstaller<T> installer = (IntentInstaller<T>) installers.get(intent.getClass());
+        IntentInstaller<T> installer = (IntentInstaller<T>) installers.get(intent
+                .getClass());
         if (installer == null) {
             throw new IntentException("no installer for class " + intent.getClass());
         }
         return installer;
     }
 
-    private <T extends Intent> List<InstallableIntent> compileIntent(T intent) {
+    private <T extends Intent> void executeCompilingPhase(T intent) {
+        setState(intent, IntentState.COMPILING);
         try {
             // For the fake, we compile using a single level pass
             List<InstallableIntent> installable = new ArrayList<>();
             for (Intent compiled : getCompiler(intent).compile(intent)) {
                 installable.add((InstallableIntent) compiled);
             }
-            setState(intent, IntentState.COMPILED);
-            return installable;
+            executeInstallingPhase(intent, installable);
+
         } catch (IntentException e) {
             setState(intent, IntentState.FAILED);
-            throw e;
+            dispatch(new IntentEvent(IntentEvent.Type.FAILED, intent));
         }
     }
 
-    private void installIntents(Intent intent, List<InstallableIntent> installable) {
+    private void executeInstallingPhase(Intent intent,
+                                        List<InstallableIntent> installable) {
+        setState(intent, IntentState.INSTALLING);
         try {
             for (InstallableIntent ii : installable) {
                 registerSubclassInstallerIfNeeded(ii);
                 getInstaller(ii).install(ii);
             }
             setState(intent, IntentState.INSTALLED);
-            putInstallable(intent.getId(), installable);
+            putInstallable(intent.id(), installable);
+            dispatch(new IntentEvent(IntentEvent.Type.INSTALLED, intent));
+
         } catch (IntentException e) {
             setState(intent, IntentState.FAILED);
-            throw e;
+            dispatch(new IntentEvent(IntentEvent.Type.FAILED, intent));
         }
     }
 
-    private void uninstallIntents(Intent intent, List<InstallableIntent> installable) {
+    private void executeWithdrawingPhase(Intent intent,
+                                         List<InstallableIntent> installable) {
+        setState(intent, IntentState.WITHDRAWING);
         try {
             for (InstallableIntent ii : installable) {
                 getInstaller(ii).uninstall(ii);
             }
+            removeInstallable(intent.id());
             setState(intent, IntentState.WITHDRAWN);
-            removeInstallable(intent.getId());
+            dispatch(new IntentEvent(IntentEvent.Type.WITHDRAWN, intent));
         } catch (IntentException e) {
+            // FIXME: Rework this to always go from WITHDRAWING to WITHDRAWN!
             setState(intent, IntentState.FAILED);
-            throw e;
+            dispatch(new IntentEvent(IntentEvent.Type.FAILED, intent));
         }
     }
 
-
     // Sets the internal state for the given intent and dispatches an event
     private void setState(Intent intent, IntentState state) {
-        IntentState previous = intentStates.get(intent.getId());
-        intentStates.put(intent.getId(), state);
-        dispatch(new IntentEvent(intent, state, previous, System.currentTimeMillis()));
+        intentStates.put(intent.id(), state);
     }
 
     private void putInstallable(IntentId id, List<InstallableIntent> installable) {
@@ -152,15 +158,15 @@ public class FakeIntentManager implements TestableIntentService {
 
     @Override
     public void submit(Intent intent) {
-        intents.put(intent.getId(), intent);
+        intents.put(intent.id(), intent);
         setState(intent, IntentState.SUBMITTED);
+        dispatch(new IntentEvent(IntentEvent.Type.SUBMITTED, intent));
         executeSubmit(intent);
     }
 
     @Override
     public void withdraw(Intent intent) {
-        intents.remove(intent.getId());
-        setState(intent, IntentState.WITHDRAWING);
+        intents.remove(intent.id());
         executeWithdraw(intent);
     }
 
@@ -175,6 +181,11 @@ public class FakeIntentManager implements TestableIntentService {
     }
 
     @Override
+    public long getIntentCount() {
+        return intents.size();
+    }
+
+    @Override
     public Intent getIntent(IntentId id) {
         return intents.get(id);
     }
@@ -185,23 +196,24 @@ public class FakeIntentManager implements TestableIntentService {
     }
 
     @Override
-    public void addListener(IntentEventListener listener) {
+    public void addListener(IntentListener listener) {
         listeners.add(listener);
     }
 
     @Override
-    public void removeListener(IntentEventListener listener) {
+    public void removeListener(IntentListener listener) {
         listeners.remove(listener);
     }
 
     private void dispatch(IntentEvent event) {
-        for (IntentEventListener listener : listeners) {
+        for (IntentListener listener : listeners) {
             listener.event(event);
         }
     }
 
     @Override
-    public <T extends Intent> void registerCompiler(Class<T> cls, IntentCompiler<T> compiler) {
+    public <T extends Intent> void registerCompiler(Class<T> cls,
+            IntentCompiler<T> compiler) {
         compilers.put(cls, compiler);
     }
 
@@ -216,7 +228,8 @@ public class FakeIntentManager implements TestableIntentService {
     }
 
     @Override
-    public <T extends InstallableIntent> void registerInstaller(Class<T> cls, IntentInstaller<T> installer) {
+    public <T extends InstallableIntent> void registerInstaller(Class<T> cls,
+            IntentInstaller<T> installer) {
         installers.put(cls, installer);
     }
 
@@ -227,7 +240,7 @@ public class FakeIntentManager implements TestableIntentService {
 
     @Override
     public Map<Class<? extends InstallableIntent>,
-            IntentInstaller<? extends InstallableIntent>> getInstallers() {
+    IntentInstaller<? extends InstallableIntent>> getInstallers() {
         return Collections.unmodifiableMap(installers);
     }
 
@@ -252,7 +265,8 @@ public class FakeIntentManager implements TestableIntentService {
         if (!installers.containsKey(intent.getClass())) {
             Class<?> cls = intent.getClass();
             while (cls != Object.class) {
-                // As long as we're within the InstallableIntent class descendants
+                // As long as we're within the InstallableIntent class
+                // descendants
                 if (InstallableIntent.class.isAssignableFrom(cls)) {
                     IntentInstaller<?> installer = installers.get(cls);
                     if (installer != null) {

@@ -10,11 +10,9 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
-import static org.onlab.onos.net.intent.IntentState.*;
 import static org.junit.Assert.*;
+import static org.onlab.onos.net.intent.IntentEvent.Type.*;
 
-// TODO: consider make it categorized as integration test when it become
-//   slow test or fragile test
 /**
  * Suite of tests for the intent service contract.
  */
@@ -51,7 +49,7 @@ public class IntentServiceTest {
     @Test
     public void basics() {
         // Make sure there are no intents
-        assertEquals("incorrect intent count", 0, service.getIntents().size());
+        assertEquals("incorrect intent count", 0, service.getIntentCount());
 
         // Register a compiler and an installer both setup for success.
         service.registerCompiler(TestIntent.class, new TestCompiler(new TestInstallableIntent(INSTALLABLE_IID)));
@@ -64,17 +62,16 @@ public class IntentServiceTest {
         TestTools.assertAfter(GRACE_MS, new Runnable() {
             @Override
             public void run() {
-                assertEquals("incorrect intent state", INSTALLED,
-                             service.getIntentState(intent.getId()));
+                assertEquals("incorrect intent state", IntentState.INSTALLED,
+                             service.getIntentState(intent.id()));
             }
         });
 
         // Make sure that all expected events have been emitted
-        validateEvents(intent, SUBMITTED, COMPILED, INSTALLED);
+        validateEvents(intent, SUBMITTED, INSTALLED);
 
         // Make sure there is just one intent (and is ours)
-        assertEquals("incorrect intent count", 1, service.getIntents().size());
-        assertEquals("incorrect intent", intent, service.getIntent(intent.getId()));
+        assertEquals("incorrect intent count", 1, service.getIntentCount());
 
         // Reset the listener events
         listener.events.clear();
@@ -86,19 +83,19 @@ public class IntentServiceTest {
         TestTools.assertAfter(GRACE_MS, new Runnable() {
             @Override
             public void run() {
-                assertEquals("incorrect intent state", WITHDRAWN,
-                             service.getIntentState(intent.getId()));
+                assertEquals("incorrect intent state", IntentState.WITHDRAWN,
+                             service.getIntentState(intent.id()));
             }
         });
 
         // Make sure that all expected events have been emitted
-        validateEvents(intent, WITHDRAWING, WITHDRAWN);
+        validateEvents(intent, WITHDRAWN);
 
         // TODO: discuss what is the fate of intents after they have been withdrawn
         // Make sure that the intent is no longer in the system
 //        assertEquals("incorrect intent count", 0, service.getIntents().size());
-//        assertNull("intent should not be found", service.getIntent(intent.getId()));
-//        assertNull("intent state should not be found", service.getIntentState(intent.getId()));
+//        assertNull("intent should not be found", service.getIntent(intent.id()));
+//        assertNull("intent state should not be found", service.getIntentState(intent.id()));
     }
 
     @Test
@@ -114,8 +111,8 @@ public class IntentServiceTest {
         TestTools.assertAfter(GRACE_MS, new Runnable() {
             @Override
             public void run() {
-                assertEquals("incorrect intent state", FAILED,
-                             service.getIntentState(intent.getId()));
+                assertEquals("incorrect intent state", IntentState.FAILED,
+                             service.getIntentState(intent.id()));
             }
         });
 
@@ -137,13 +134,13 @@ public class IntentServiceTest {
         TestTools.assertAfter(GRACE_MS, new Runnable() {
             @Override
             public void run() {
-                assertEquals("incorrect intent state", FAILED,
-                             service.getIntentState(intent.getId()));
+                assertEquals("incorrect intent state", IntentState.FAILED,
+                             service.getIntentState(intent.id()));
             }
         });
 
         // Make sure that all expected events have been emitted
-        validateEvents(intent, SUBMITTED, COMPILED, FAILED);
+        validateEvents(intent, SUBMITTED, FAILED);
     }
 
     /**
@@ -152,23 +149,23 @@ public class IntentServiceTest {
      * considered.
      *
      * @param intent intent subject
-     * @param states list of states for which events are expected
+     * @param types  list of event types for which events are expected
      */
-    protected void validateEvents(Intent intent, IntentState... states) {
+    protected void validateEvents(Intent intent, IntentEvent.Type... types) {
         Iterator<IntentEvent> events = listener.events.iterator();
-        for (IntentState state : states) {
+        for (IntentEvent.Type type : types) {
             IntentEvent event = events.hasNext() ? events.next() : null;
             if (event == null) {
-                fail("expected event not found: " + state);
-            } else if (intent.equals(event.getIntent())) {
-                assertEquals("incorrect state", state, event.getState());
+                fail("expected event not found: " + type);
+            } else if (intent.equals(event.subject())) {
+                assertEquals("incorrect state", type, event.type());
             }
         }
 
         // Remainder of events should not apply to this intent; make sure.
         while (events.hasNext()) {
             assertFalse("unexpected event for intent",
-                        intent.equals(events.next().getIntent()));
+                        intent.equals(events.next().subject()));
         }
     }
 
@@ -229,8 +226,8 @@ public class IntentServiceTest {
         TestTools.assertAfter(GRACE_MS, new Runnable() {
             @Override
             public void run() {
-                assertEquals("incorrect intent state", INSTALLED,
-                             service.getIntentState(intent.getId()));
+                assertEquals("incorrect intent state", IntentState.INSTALLED,
+                             service.getIntentState(intent.id()));
             }
         });
 
@@ -250,7 +247,7 @@ public class IntentServiceTest {
 
 
     // Fixture to track emitted intent events
-    protected class TestListener implements IntentEventListener {
+    protected class TestListener implements IntentListener {
         final List<IntentEvent> events = new ArrayList<>();
 
         @Override

@@ -12,10 +12,10 @@ public class MultiPointToSinglePointIntentTest extends ConnectivityIntentTest {
     @Test
     public void basics() {
         MultiPointToSinglePointIntent intent = createOne();
-        assertEquals("incorrect id", IID, intent.getId());
-        assertEquals("incorrect match", MATCH, intent.getTrafficSelector());
-        assertEquals("incorrect ingress", PS1, intent.getIngressPorts());
-        assertEquals("incorrect egress", P2, intent.getEgressPort());
+        assertEquals("incorrect id", IID, intent.id());
+        assertEquals("incorrect match", MATCH, intent.selector());
+        assertEquals("incorrect ingress", PS1, intent.ingressPoints());
+        assertEquals("incorrect egress", P2, intent.egressPoint());
     }
 
     @Override

@@ -16,12 +16,12 @@ public class PathIntentTest extends ConnectivityIntentTest {
     @Test
     public void basics() {
         PathIntent intent = createOne();
-        assertEquals("incorrect id", IID, intent.getId());
-        assertEquals("incorrect match", MATCH, intent.getTrafficSelector());
-        assertEquals("incorrect action", NOP, intent.getTrafficTreatment());
-        assertEquals("incorrect ingress", P1, intent.getIngressPort());
-        assertEquals("incorrect egress", P2, intent.getEgressPort());
-        assertEquals("incorrect path", PATH1, intent.getPath());
+        assertEquals("incorrect id", IID, intent.id());
+        assertEquals("incorrect match", MATCH, intent.selector());
+        assertEquals("incorrect action", NOP, intent.treatment());
+        assertEquals("incorrect ingress", P1, intent.ingressPoint());
+        assertEquals("incorrect egress", P2, intent.egressPoint());
+        assertEquals("incorrect path", PATH1, intent.path());
     }
 
     @Override

@@ -12,10 +12,10 @@ public class PointToPointIntentTest extends ConnectivityIntentTest {
     @Test
     public void basics() {
         PointToPointIntent intent = createOne();
-        assertEquals("incorrect id", IID, intent.getId());
-        assertEquals("incorrect match", MATCH, intent.getTrafficSelector());
-        assertEquals("incorrect ingress", P1, intent.getIngressPort());
-        assertEquals("incorrect egress", P2, intent.getEgressPort());
+        assertEquals("incorrect id", IID, intent.id());
+        assertEquals("incorrect match", MATCH, intent.selector());
+        assertEquals("incorrect ingress", P1, intent.ingressPoint());
+        assertEquals("incorrect egress", P2, intent.egressPoint());
     }
 
     @Override

@@ -12,10 +12,10 @@ public class SinglePointToMultiPointIntentTest extends ConnectivityIntentTest {
     @Test
     public void basics() {
         SinglePointToMultiPointIntent intent = createOne();
-        assertEquals("incorrect id", IID, intent.getId());
-        assertEquals("incorrect match", MATCH, intent.getTrafficSelector());
-        assertEquals("incorrect ingress", P1, intent.getIngressPort());
-        assertEquals("incorrect egress", PS2, intent.getEgressPorts());
+        assertEquals("incorrect id", IID, intent.id());
+        assertEquals("incorrect match", MATCH, intent.selector());
+        assertEquals("incorrect ingress", P1, intent.ingressPoint());
+        assertEquals("incorrect egress", PS2, intent.egressPoints());
     }
 
     @Override

 package org.onlab.onos.net.intent;
 //TODO is this the right package?
 
+import org.onlab.onos.net.Link;
+
+import java.util.Collection;
+
 /**
  * An installable intent used in the unit test.
  *
@@ -25,4 +29,8 @@ public class TestInstallableIntent extends AbstractIntent implements Installable
         super();
     }
 
+    @Override
+    public Collection<Link> requiredLinks() {
+        return null;
+    }
 }

@@ -37,5 +37,4 @@ public class DefaultGraphDescriptionTest {
         new DefaultGraphDescription(4321L, ImmutableSet.of(DEV1, DEV3),
                                     ImmutableSet.of(L1, L2));
     }
-
-}
\ No newline at end of file
+}

@@ -50,5 +50,4 @@ public class DefaultTopologyEdgeTest {
                                   new DefaultTopologyEdge(V2, V1, L2))
                 .testEquals();
     }
-
-}
\ No newline at end of file
+}

@@ -26,5 +26,4 @@ public class DefaultTopologyVertexTest {
                 .addEqualityGroup(new DefaultTopologyVertex(D2),
                                   new DefaultTopologyVertex(D2)).testEquals();
     }
-
-}
\ No newline at end of file
+}

@@ -36,6 +36,12 @@
             <scope>test</scope>
         </dependency>
 
+	<dependency>
+	    <groupId>org.easymock</groupId>
+	    <artifactId>easymock</artifactId>
+	    <scope>test</scope>
+	</dependency>
+
         <!-- TODO Consider removing store dependency.
               Currently required for DistributedDeviceManagerTest. -->
         <dependency>

+package org.onlab.onos.cluster.impl;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.CoreService;
+import org.onlab.onos.Version;
+import org.onlab.util.Tools;
+
+import java.io.File;
+import java.util.List;
+
+/**
+ * Core service implementation.
+ */
+@Component
+@Service
+public class CoreManager implements CoreService {
+
+    private static final File VERSION_FILE = new File("../VERSION");
+    private static Version version = Version.version("1.0.0-SNAPSHOT");
+
+    // TODO: work in progress
+
+    @Activate
+    public void activate() {
+        List<String> versionLines = Tools.slurp(VERSION_FILE);
+        if (versionLines != null && !versionLines.isEmpty()) {
+            version = Version.version(versionLines.get(0));
+        }
+    }
+
+    @Override
+    public Version version() {
+        return version;
+    }
+
+}

@@ -18,6 +18,7 @@ import org.onlab.onos.cluster.MastershipListener;
 import org.onlab.onos.cluster.MastershipService;
 import org.onlab.onos.cluster.MastershipTermService;
 import org.onlab.onos.cluster.MastershipTerm;
+import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.event.AbstractListenerRegistry;
 import org.onlab.onos.event.EventDeliveryService;
 import org.onlab.onos.net.Device;
@@ -38,7 +39,7 @@ import org.onlab.onos.net.device.DeviceStoreDelegate;
 import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.AbstractProviderRegistry;
 import org.onlab.onos.net.provider.AbstractProviderService;
-import org.onlab.onos.store.ClockService;
+import org.onlab.onos.store.ClockProviderService;
 import org.slf4j.Logger;
 
 /**
@@ -80,7 +81,7 @@ public class DeviceManager
     protected MastershipTermService termService;
 
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected ClockService clockService;
+    protected ClockProviderService clockProviderService;
 
     @Activate
     public void activate() {
@@ -144,6 +145,10 @@ public class DeviceManager
     private void applyRole(DeviceId deviceId, MastershipRole newRole) {
         if (newRole.equals(MastershipRole.NONE)) {
             Device device = store.getDevice(deviceId);
+            // FIXME: Device might not be there yet. (eventual consistent)
+            if (device == null) {
+                return;
+            }
             DeviceProvider provider = getProvider(device.providerId());
             if (provider != null) {
                 provider.roleChanged(device, newRole);
@@ -193,13 +198,50 @@ public class DeviceManager
             checkNotNull(deviceId, DEVICE_ID_NULL);
             checkNotNull(deviceDescription, DEVICE_DESCRIPTION_NULL);
             checkValidity();
+
+            log.info("Device {} connected", deviceId);
+            // check my Role
+            MastershipRole role = mastershipService.requestRoleFor(deviceId);
+
+            if (role != MastershipRole.MASTER) {
+                // TODO: Do we need to explicitly tell the Provider that
+                // this instance is no longer the MASTER? probably not
+                return;
+            }
+
+            MastershipTerm term = mastershipService.requestTermService()
+                    .getMastershipTerm(deviceId);
+            if (!term.master().equals(clusterService.getLocalNode().id())) {
+                // lost mastership after requestRole told this instance was MASTER.
+                return;
+            }
+            // tell clock provider if this instance is the master
+            clockProviderService.setMastershipTerm(deviceId, term);
+
             DeviceEvent event = store.createOrUpdateDevice(provider().id(),
                     deviceId, deviceDescription);
 
+            // If there was a change of any kind, tell the provider
+            // that this instance is the master.
+            // Note: event can be null, if mastership was lost between
+            // roleRequest and store update calls.
             if (event != null) {
-                log.info("Device {} connected", deviceId);
-                provider().roleChanged(event.subject(),
-                        mastershipService.requestRoleFor(deviceId));
+                // TODO: Check switch reconnected case. Is it assured that
+                //       event will never be null?
+                //       Could there be a situation MastershipService told this
+                //       instance is the new Master, but
+                //       event returned from the store is null?
+
+                // TODO: Confirm: Mastership could be lost after requestRole
+                //       and createOrUpdateDevice call.
+                //       In that case STANDBY node can
+                //       claim itself to be master against the Device.
+                //       Will the Node, chosen by the MastershipService, retry
+                //       to get the MASTER role when that happen?
+
+                // FIXME: 1st argument should be deviceId, to allow setting
+                //        certain roles even if the store returned null.
+                provider().roleChanged(event.subject(), role);
                 post(event);
             }
         }
@@ -208,6 +250,15 @@ public class DeviceManager
         public void deviceDisconnected(DeviceId deviceId) {
             checkNotNull(deviceId, DEVICE_ID_NULL);
             checkValidity();
+
+            // FIXME: only the MASTER should be marking off-line in normal cases,
+            // but if I was the last STANDBY connection, etc. and no one else
+            // was there to mark the device offline, this instance may need to
+            // temporarily request for Master Role and mark offline.
+            if (!mastershipService.getLocalRole(deviceId).equals(MastershipRole.MASTER)) {
+                log.debug("Device {} disconnected, but I am not the master", deviceId);
+                return;
+            }
             DeviceEvent event = store.markOffline(deviceId);
             //we're no longer capable of being master or a candidate.
             mastershipService.relinquishMastership(deviceId);
@@ -253,6 +304,9 @@ public class DeviceManager
             // FIXME: implement response to this notification
             log.warn("Failed to assert role [{}] onto Device {}", role,
                     deviceId);
+            if (role == MastershipRole.MASTER) {
+                mastershipService.relinquishMastership(deviceId);
+            }
         }
     }
 
@@ -268,11 +322,17 @@ public class DeviceManager
 
         @Override
         public void event(MastershipEvent event) {
-            DeviceId did = event.subject();
+            final DeviceId did = event.subject();
             if (isAvailable(did)) {
-                if (event.master().equals(clusterService.getLocalNode().id())) {
+                final NodeId myNodeId = clusterService.getLocalNode().id();
+
+                if (myNodeId.equals(event.master())) {
                     MastershipTerm term = termService.getMastershipTerm(did);
-                    clockService.setMastershipTerm(did, term);
+
+                    if (term.master().equals(myNodeId)) {
+                        // only set the new term if I am the master
+                        clockProviderService.setMastershipTerm(did, term);
+                    }
                     applyRole(did, MastershipRole.MASTER);
                 } else {
                     applyRole(did, MastershipRole.STANDBY);

@@ -5,9 +5,10 @@ import static org.slf4j.LoggerFactory.getLogger;
 
 import java.util.Iterator;
 import java.util.List;
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
 
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
@@ -21,7 +22,11 @@ import org.onlab.onos.event.EventDeliveryService;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.device.DeviceService;
+import org.onlab.onos.net.flow.CompletedBatchOperation;
+import org.onlab.onos.net.flow.FlowEntry;
 import org.onlab.onos.net.flow.FlowRule;
+import org.onlab.onos.net.flow.FlowRuleBatchEntry;
+import org.onlab.onos.net.flow.FlowRuleBatchOperation;
 import org.onlab.onos.net.flow.FlowRuleEvent;
 import org.onlab.onos.net.flow.FlowRuleListener;
 import org.onlab.onos.net.flow.FlowRuleProvider;
@@ -34,7 +39,9 @@ import org.onlab.onos.net.provider.AbstractProviderRegistry;
 import org.onlab.onos.net.provider.AbstractProviderService;
 import org.slf4j.Logger;
 
+import com.google.common.collect.ArrayListMultimap;
 import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
 
 /**
  * Provides implementation of the flow NB & SB APIs.
@@ -42,14 +49,14 @@ import com.google.common.collect.Lists;
 @Component(immediate = true)
 @Service
 public class FlowRuleManager
-extends AbstractProviderRegistry<FlowRuleProvider, FlowRuleProviderService>
-implements FlowRuleService, FlowRuleProviderRegistry {
+        extends AbstractProviderRegistry<FlowRuleProvider, FlowRuleProviderService>
+        implements FlowRuleService, FlowRuleProviderRegistry {
 
     public static final String FLOW_RULE_NULL = "FlowRule cannot be null";
     private final Logger log = getLogger(getClass());
 
     private final AbstractListenerRegistry<FlowRuleEvent, FlowRuleListener>
-    listenerRegistry = new AbstractListenerRegistry<>();
+            listenerRegistry = new AbstractListenerRegistry<>();
 
     private final FlowRuleStoreDelegate delegate = new InternalStoreDelegate();
 
@@ -62,8 +69,6 @@ implements FlowRuleService, FlowRuleProviderRegistry {
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected DeviceService deviceService;
 
-    private final Map<FlowRule, AtomicInteger> deadRounds = new ConcurrentHashMap<>();
-
     @Activate
     public void activate() {
         store.setDelegate(delegate);
@@ -79,7 +84,12 @@ implements FlowRuleService, FlowRuleProviderRegistry {
     }
 
     @Override
-    public Iterable<FlowRule> getFlowEntries(DeviceId deviceId) {
+    public int getFlowRuleCount() {
+        return store.getFlowRuleCount();
+    }
+
+    @Override
+    public Iterable<FlowEntry> getFlowEntries(DeviceId deviceId) {
         return store.getFlowEntries(deviceId);
     }
 
@@ -89,7 +99,6 @@ implements FlowRuleService, FlowRuleProviderRegistry {
             FlowRule f = flowRules[i];
             final Device device = deviceService.getDevice(f.deviceId());
             final FlowRuleProvider frp = getProvider(device.providerId());
-            deadRounds.put(f, new AtomicInteger(0));
             store.storeFlowRule(f);
             frp.applyFlowRule(f);
         }
@@ -103,16 +112,17 @@ implements FlowRuleService, FlowRuleProviderRegistry {
         for (int i = 0; i < flowRules.length; i++) {
             f = flowRules[i];
             device = deviceService.getDevice(f.deviceId());
-            frp = getProvider(device.providerId());
-            deadRounds.remove(f);
             store.deleteFlowRule(f);
-            frp.removeFlowRule(f);
+            if (device != null) {
+                frp = getProvider(device.providerId());
+                frp.removeFlowRule(f);
+            }
         }
     }
 
     @Override
     public void removeFlowRulesById(ApplicationId id) {
-        Iterable<FlowRule> rules =  getFlowRulesById(id);
+        Iterable<FlowRule> rules = getFlowRulesById(id);
         FlowRuleProvider frp;
         Device device;
 
@@ -126,7 +136,39 @@ implements FlowRuleService, FlowRuleProviderRegistry {
 
     @Override
     public Iterable<FlowRule> getFlowRulesById(ApplicationId id) {
-        return store.getFlowEntriesByAppId(id);
+        return store.getFlowRulesByAppId(id);
+    }
+
+    @Override
+    public Future<CompletedBatchOperation> applyBatch(
+            FlowRuleBatchOperation batch) {
+        Multimap<FlowRuleProvider, FlowRuleBatchEntry> batches =
+                ArrayListMultimap.create();
+        List<Future<Void>> futures = Lists.newArrayList();
+        for (FlowRuleBatchEntry fbe : batch.getOperations()) {
+            final FlowRule f = fbe.getTarget();
+            final Device device = deviceService.getDevice(f.deviceId());
+            final FlowRuleProvider frp = getProvider(device.providerId());
+            batches.put(frp, fbe);
+            switch (fbe.getOperator()) {
+                case ADD:
+                    store.storeFlowRule(f);
+                    break;
+                case REMOVE:
+                    store.deleteFlowRule(f);
+                    break;
+                case MODIFY:
+                default:
+                    log.error("Batch operation type {} unsupported.", fbe.getOperator());
+            }
+        }
+        for (FlowRuleProvider provider : batches.keySet()) {
+            FlowRuleBatchOperation b =
+                    new FlowRuleBatchOperation(batches.get(provider));
+            Future<Void> future = provider.executeBatch(b);
+            futures.add(future);
+        }
+        return new FlowRuleBatchFuture(futures);
     }
 
     @Override
@@ -146,63 +188,63 @@ implements FlowRuleService, FlowRuleProviderRegistry {
     }
 
     private class InternalFlowRuleProviderService
-    extends AbstractProviderService<FlowRuleProvider>
-    implements FlowRuleProviderService {
+            extends AbstractProviderService<FlowRuleProvider>
+            implements FlowRuleProviderService {
 
         protected InternalFlowRuleProviderService(FlowRuleProvider provider) {
             super(provider);
         }
 
         @Override
-        public void flowRemoved(FlowRule flowRule) {
-            checkNotNull(flowRule, FLOW_RULE_NULL);
+        public void flowRemoved(FlowEntry flowEntry) {
+            checkNotNull(flowEntry, FLOW_RULE_NULL);
             checkValidity();
-            FlowRule stored = store.getFlowRule(flowRule);
+            FlowEntry stored = store.getFlowEntry(flowEntry);
             if (stored == null) {
-                log.debug("Rule already evicted from store: {}", flowRule);
+                log.info("Rule already evicted from store: {}", flowEntry);
                 return;
             }
-            Device device = deviceService.getDevice(flowRule.deviceId());
+            Device device = deviceService.getDevice(flowEntry.deviceId());
             FlowRuleProvider frp = getProvider(device.providerId());
             FlowRuleEvent event = null;
             switch (stored.state()) {
-            case ADDED:
-            case PENDING_ADD:
+                case ADDED:
+                case PENDING_ADD:
                     frp.applyFlowRule(stored);
-                break;
-            case PENDING_REMOVE:
-            case REMOVED:
-                event = store.removeFlowRule(flowRule);
-                break;
-            default:
-                break;
+                    break;
+                case PENDING_REMOVE:
+                case REMOVED:
+                    event = store.removeFlowRule(stored);
+                    break;
+                default:
+                    break;
 
             }
             if (event != null) {
-                log.debug("Flow {} removed", flowRule);
+                log.debug("Flow {} removed", flowEntry);
                 post(event);
             }
         }
 
 
-        private void flowMissing(FlowRule flowRule) {
+        private void flowMissing(FlowEntry flowRule) {
             checkNotNull(flowRule, FLOW_RULE_NULL);
             checkValidity();
             Device device = deviceService.getDevice(flowRule.deviceId());
             FlowRuleProvider frp = getProvider(device.providerId());
             FlowRuleEvent event = null;
             switch (flowRule.state()) {
-            case PENDING_REMOVE:
-            case REMOVED:
-                event = store.removeFlowRule(flowRule);
-                frp.removeFlowRule(flowRule);
-                break;
-            case ADDED:
-            case PENDING_ADD:
-                frp.applyFlowRule(flowRule);
-                break;
-            default:
-                log.debug("Flow {} has not been installed.", flowRule);
+                case PENDING_REMOVE:
+                case REMOVED:
+                    event = store.removeFlowRule(flowRule);
+                    frp.removeFlowRule(flowRule);
+                    break;
+                case ADDED:
+                case PENDING_ADD:
+                    frp.applyFlowRule(flowRule);
+                    break;
+                default:
+                    log.debug("Flow {} has not been installed.", flowRule);
             }
 
             if (event != null) {
@@ -221,36 +263,40 @@ implements FlowRuleService, FlowRuleProviderRegistry {
         }
 
 
-        private void flowAdded(FlowRule flowRule) {
-            checkNotNull(flowRule, FLOW_RULE_NULL);
+        private void flowAdded(FlowEntry flowEntry) {
+            checkNotNull(flowEntry, FLOW_RULE_NULL);
             checkValidity();
 
-            if (deadRounds.containsKey(flowRule) &&
-                    checkRuleLiveness(flowRule, store.getFlowRule(flowRule))) {
+            if (checkRuleLiveness(flowEntry, store.getFlowEntry(flowEntry))) {
 
-                FlowRuleEvent event = store.addOrUpdateFlowRule(flowRule);
+                FlowRuleEvent event = store.addOrUpdateFlowRule(flowEntry);
                 if (event == null) {
                     log.debug("No flow store event generated.");
                 } else {
-                    log.debug("Flow {} {}", flowRule, event.type());
+                    log.debug("Flow {} {}", flowEntry, event.type());
                     post(event);
                 }
             } else {
-                removeFlowRules(flowRule);
+                removeFlowRules(flowEntry);
             }
 
         }
 
-        private boolean checkRuleLiveness(FlowRule swRule, FlowRule storedRule) {
-            int timeout = storedRule.timeout();
+        private boolean checkRuleLiveness(FlowEntry swRule, FlowEntry storedRule) {
+            if (storedRule == null) {
+                return false;
+            }
+            long timeout = storedRule.timeout() * 1000;
+            Long currentTime = System.currentTimeMillis();
             if (storedRule.packets() != swRule.packets()) {
-                deadRounds.get(swRule).set(0);
+                storedRule.setLastSeen();
                 return true;
             }
 
-            return (deadRounds.get(swRule).getAndIncrement() *
-                    FlowRuleProvider.POLL_INTERVAL) <= timeout;
-
+            if ((currentTime - storedRule.lastSeen()) <= timeout) {
+                return true;
+            }
+            return false;
         }
 
         // Posts the specified event to the local event dispatcher.
@@ -261,13 +307,13 @@ implements FlowRuleService, FlowRuleProviderRegistry {
         }
 
         @Override
-        public void pushFlowMetrics(DeviceId deviceId, Iterable<FlowRule> flowEntries) {
-            List<FlowRule> storedRules = Lists.newLinkedList(store.getFlowEntries(deviceId));
+        public void pushFlowMetrics(DeviceId deviceId, Iterable<FlowEntry> flowEntries) {
+            List<FlowEntry> storedRules = Lists.newLinkedList(store.getFlowEntries(deviceId));
 
-            Iterator<FlowRule> switchRulesIterator = flowEntries.iterator();
+            Iterator<FlowEntry> switchRulesIterator = flowEntries.iterator();
 
             while (switchRulesIterator.hasNext()) {
-                FlowRule rule = switchRulesIterator.next();
+                FlowEntry rule = switchRulesIterator.next();
                 if (storedRules.remove(rule)) {
                     // we both have the rule, let's update some info then.
                     flowAdded(rule);
@@ -276,7 +322,7 @@ implements FlowRuleService, FlowRuleProviderRegistry {
                     extraneousFlow(rule);
                 }
             }
-            for (FlowRule rule : storedRules) {
+            for (FlowEntry rule : storedRules) {
                 // there are rules in the store that aren't on the switch
                 flowMissing(rule);
 
@@ -291,4 +337,63 @@ implements FlowRuleService, FlowRuleProviderRegistry {
             eventDispatcher.post(event);
         }
     }
+
+    private class FlowRuleBatchFuture
+        implements Future<CompletedBatchOperation> {
+
+        private final List<Future<Void>> futures;
+
+        public FlowRuleBatchFuture(List<Future<Void>> futures) {
+            this.futures = futures;
+        }
+
+        @Override
+        public boolean cancel(boolean mayInterruptIfRunning) {
+            // TODO Auto-generated method stub
+            return false;
+        }
+
+        @Override
+        public boolean isCancelled() {
+            // TODO Auto-generated method stub
+            return false;
+        }
+
+        @Override
+        public boolean isDone() {
+            boolean isDone = true;
+            for (Future<Void> future : futures) {
+                isDone &= future.isDone();
+            }
+            return isDone;
+        }
+
+        @Override
+        public CompletedBatchOperation get() throws InterruptedException,
+        ExecutionException {
+            // TODO Auto-generated method stub
+            for (Future<Void> future : futures) {
+                future.get();
+            }
+            return new CompletedBatchOperation();
+        }
+
+        @Override
+        public CompletedBatchOperation get(long timeout, TimeUnit unit)
+                throws InterruptedException, ExecutionException,
+                TimeoutException {
+            // TODO we should decrement the timeout
+            long start = System.nanoTime();
+            long end = start + unit.toNanos(timeout);
+            for (Future<Void> future : futures) {
+                long now = System.nanoTime();
+                long thisTimeout = end - now;
+                future.get(thisTimeout, TimeUnit.NANOSECONDS);
+            }
+            return new CompletedBatchOperation();
+        }
+
+    }
+
+
 }

@@ -76,7 +76,7 @@ public class HostManager
         eventDispatcher.addSink(HostEvent.class, listenerRegistry);
 
         monitor = new HostMonitor(deviceService,  packetService, this);
-
+        monitor.start();
     }
 
     @Deactivate

@@ -2,11 +2,11 @@ package org.onlab.onos.net.host.impl;
 
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
-import java.util.HashSet;
+import java.util.Collections;
 import java.util.List;
-import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 import java.util.concurrent.TimeUnit;
 
 import org.jboss.netty.util.Timeout;
@@ -33,8 +33,6 @@ import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.util.Timer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Monitors hosts on the dataplane to detect changes in host data.
@@ -44,70 +42,91 @@ import org.slf4j.LoggerFactory;
  * probe for hosts that have not yet been detected (specified by IP address).
  */
 public class HostMonitor implements TimerTask {
-    private static final Logger log = LoggerFactory.getLogger(HostMonitor.class);
-
-    private static final byte[] ZERO_MAC_ADDRESS =
-            MacAddress.valueOf("00:00:00:00:00:00").getAddress();
-
-    // TODO put on Ethernet
-    private static final byte[] BROADCAST_MAC =
-            MacAddress.valueOf("ff:ff:ff:ff:ff:ff").getAddress();
-
     private DeviceService deviceService;
     private PacketService packetService;
     private HostManager hostManager;
 
     private final Set<IpAddress> monitoredAddresses;
 
-    private final Map<ProviderId, HostProvider> hostProviders;
+    private final ConcurrentMap<ProviderId, HostProvider> hostProviders;
 
-    private final long probeRate;
+    private static final long DEFAULT_PROBE_RATE = 30000; // milliseconds
+    private long probeRate = DEFAULT_PROBE_RATE;
 
-    private final Timeout timeout;
+    private Timeout timeout;
 
-    public HostMonitor(
-            DeviceService deviceService,
-            PacketService packetService,
-            HostManager hostService) {
+    /**
+     * Creates a new host monitor.
+     *
+     * @param deviceService device service used to find edge ports
+     * @param packetService packet service used to send packets on the data plane
+     * @param hostManager host manager used to look up host information and
+     * probe existing hosts
+     */
+    public HostMonitor(DeviceService deviceService, PacketService packetService,
+            HostManager hostManager) {
 
         this.deviceService = deviceService;
         this.packetService = packetService;
-        this.hostManager = hostService;
+        this.hostManager = hostManager;
 
-        monitoredAddresses = new HashSet<>();
+        monitoredAddresses = Collections.newSetFromMap(
+                new ConcurrentHashMap<IpAddress, Boolean>());
         hostProviders = new ConcurrentHashMap<>();
 
-        probeRate = 30000; // milliseconds
-
         timeout = Timer.getTimer().newTimeout(this, 0, TimeUnit.MILLISECONDS);
-
-        addDefaultAddresses();
-    }
-
-    private void addDefaultAddresses() {
-        //monitoredAddresses.add(IpAddress.valueOf("10.0.0.1"));
     }
 
+    /**
+     * Adds an IP address to be monitored by the host monitor. The monitor will
+     * periodically probe the host to detect changes.
+     *
+     * @param ip IP address of the host to monitor
+     */
     void addMonitoringFor(IpAddress ip) {
         monitoredAddresses.add(ip);
     }
 
+    /**
+     * Stops monitoring the given IP address.
+     *
+     * @param ip IP address to stop monitoring on
+     */
     void stopMonitoring(IpAddress ip) {
         monitoredAddresses.remove(ip);
     }
 
+    /**
+     * Starts the host monitor. Does nothing if the monitor is already running.
+     */
+    void start() {
+        synchronized (this) {
+            if (timeout == null) {
+                timeout = Timer.getTimer().newTimeout(this, 0, TimeUnit.MILLISECONDS);
+            }
+        }
+    }
+
+    /**
+     * Stops the host monitor.
+     */
     void shutdown() {
-        timeout.cancel();
+        synchronized (this) {
+            timeout.cancel();
+            timeout = null;
+        }
     }
 
+    /**
+     * Registers a host provider with the host monitor. The monitor can use the
+     * provider to probe hosts.
+     *
+     * @param provider the host provider to register
+     */
     void registerHostProvider(HostProvider provider) {
         hostProviders.put(provider.id(), provider);
     }
 
-    void unregisterHostProvider(HostProvider provider) {
-        // TODO find out how to call this
-    }
-
     @Override
     public void run(Timeout timeout) throws Exception {
         for (IpAddress ip : monitoredAddresses) {
@@ -121,14 +140,16 @@ public class HostMonitor implements TimerTask {
             } else {
                 for (Host host : hosts) {
                     HostProvider provider = hostProviders.get(host.providerId());
-                    if (provider != null) {
+                    if (provider == null) {
+                        hostProviders.remove(host.providerId(), null);
+                    } else {
                         provider.triggerProbe(host);
                     }
                 }
             }
         }
 
-        timeout = Timer.getTimer().newTimeout(this, probeRate, TimeUnit.MILLISECONDS);
+        this.timeout = Timer.getTimer().newTimeout(this, probeRate, TimeUnit.MILLISECONDS);
     }
 
     /**
@@ -161,7 +182,7 @@ public class HostMonitor implements TimerTask {
         List<Instruction> instructions = new ArrayList<>();
         instructions.add(Instructions.createOutput(port.number()));
 
-        TrafficTreatment treatment = new DefaultTrafficTreatment.Builder()
+        TrafficTreatment treatment = DefaultTrafficTreatment.builder()
         .setOutput(port.number())
         .build();
 
@@ -184,12 +205,12 @@ public class HostMonitor implements TimerTask {
 
         arp.setSenderHardwareAddress(sourceMac.getAddress())
            .setSenderProtocolAddress(sourceIp.toOctets())
-           .setTargetHardwareAddress(ZERO_MAC_ADDRESS)
+           .setTargetHardwareAddress(MacAddress.ZERO_MAC_ADDRESS)
            .setTargetProtocolAddress(targetIp.toOctets());
 
         Ethernet ethernet = new Ethernet();
         ethernet.setEtherType(Ethernet.TYPE_ARP)
-                .setDestinationMACAddress(BROADCAST_MAC)
+                .setDestinationMACAddress(MacAddress.BROADCAST_MAC)
                 .setSourceMACAddress(sourceMac.getAddress())
                 .setPayload(arp);
 

+package org.onlab.onos.net.intent.impl;
+
+import org.onlab.onos.net.intent.IdGenerator;
+
+/**
+ * Base class of {@link IdGenerator} implementations which use {@link IdBlockAllocator} as
+ * backend.
+ *
+ * @param <T> the type of ID
+ */
+public abstract class AbstractBlockAllocatorBasedIdGenerator<T> implements IdGenerator<T> {
+    protected final IdBlockAllocator allocator;
+    protected IdBlock idBlock;
+
+    /**
+     * Constructs an ID generator which use {@link IdBlockAllocator} as backend.
+     *
+     * @param allocator
+     */
+    protected AbstractBlockAllocatorBasedIdGenerator(IdBlockAllocator allocator) {
+        this.allocator = allocator;
+        this.idBlock = allocator.allocateUniqueIdBlock();
+    }
+
+    @Override
+    public synchronized T getNewId() {
+        try {
+            return convertFrom(idBlock.getNextId());
+        } catch (UnavailableIdException e) {
+            idBlock = allocator.allocateUniqueIdBlock();
+            return convertFrom(idBlock.getNextId());
+        }
+    }
+
+    /**
+     * Returns an ID instance of {@code T} type from the long value.
+     *
+     * @param value original long value
+     * @return ID instance
+     */
+    protected abstract T convertFrom(long value);
+}

+package org.onlab.onos.net.intent.impl;
+
+public class DummyIdBlockAllocator implements IdBlockAllocator {
+    private long blockTop;
+    private static final long BLOCK_SIZE = 0x1000000L;
+
+    /**
+     * Returns a block of IDs which are unique and unused.
+     * Range of IDs is fixed size and is assigned incrementally as this method
+     * called.
+     *
+     * @return an IdBlock containing a set of unique IDs
+     */
+    @Override
+    public IdBlock allocateUniqueIdBlock() {
+        synchronized (this)  {
+            long blockHead = blockTop;
+            long blockTail = blockTop + BLOCK_SIZE;
+
+            IdBlock block = new IdBlock(blockHead, BLOCK_SIZE);
+            blockTop = blockTail;
+
+            return block;
+        }
+    }
+
+    @Override
+    public IdBlock allocateUniqueIdBlock(long range) {
+        throw new UnsupportedOperationException("Not supported yet");
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.onlab.onos.net.Host;
+import org.onlab.onos.net.HostId;
+import org.onlab.onos.net.Path;
+import org.onlab.onos.net.flow.TrafficSelector;
+import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.intent.HostToHostIntent;
+import org.onlab.onos.net.intent.IdGenerator;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentCompiler;
+import org.onlab.onos.net.intent.IntentExtensionService;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.PathIntent;
+import org.onlab.onos.net.topology.PathService;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import static org.onlab.onos.net.flow.DefaultTrafficSelector.builder;
+
+/**
+ * A intent compiler for {@link HostToHostIntent}.
+ */
+@Component(immediate = true)
+public class HostToHostIntentCompiler
+        implements IntentCompiler<HostToHostIntent> {
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected IntentExtensionService intentManager;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected PathService pathService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostService hostService;
+
+    private IdGenerator<IntentId> intentIdGenerator;
+
+    @Activate
+    public void activate() {
+        IdBlockAllocator idBlockAllocator = new DummyIdBlockAllocator();
+        intentIdGenerator = new IdBlockAllocatorBasedIntentIdGenerator(idBlockAllocator);
+        intentManager.registerCompiler(HostToHostIntent.class, this);
+    }
+
+    @Deactivate
+    public void deactivate() {
+        intentManager.unregisterCompiler(HostToHostIntent.class);
+    }
+
+    @Override
+    public List<Intent> compile(HostToHostIntent intent) {
+        Path pathOne = getPath(intent.one(), intent.two());
+        Path pathTwo = getPath(intent.two(), intent.one());
+
+        Host one = hostService.getHost(intent.one());
+        Host two = hostService.getHost(intent.two());
+
+        return Arrays.asList(createPathIntent(pathOne, one, two, intent),
+                             createPathIntent(pathTwo, two, one, intent));
+    }
+
+    // Creates a path intent from the specified path and original connectivity intent.
+    private Intent createPathIntent(Path path, Host src, Host dst,
+                                    HostToHostIntent intent) {
+
+        TrafficSelector selector = builder(intent.selector())
+                .matchEthSrc(src.mac()).matchEthDst(dst.mac()).build();
+
+        return new PathIntent(intentIdGenerator.getNewId(),
+                              selector, intent.treatment(),
+                              path.src(), path.dst(), path);
+    }
+
+    private Path getPath(HostId one, HostId two) {
+        Set<Path> paths = pathService.getPaths(one, two);
+        if (paths.isEmpty()) {
+            throw new PathNotFoundException("No path from host " + one + " to " + two);
+        }
+        // TODO: let's be more intelligent about this eventually
+        return paths.iterator().next();
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+import java.util.Objects;
+import java.util.concurrent.atomic.AtomicLong;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * A class representing an ID space.
+ */
+public final class IdBlock {
+    private final long start;
+    private final long size;
+
+    private final AtomicLong currentId;
+
+    /**
+     * Constructs a new ID block with the specified size and initial value.
+     *
+     * @param start initial value of the block
+     * @param size size of the block
+     * @throws IllegalArgumentException if the size is less than or equal to 0
+     */
+    public IdBlock(long start, long size) {
+        checkArgument(size > 0, "size should be more than 0, but %s", size);
+
+        this.start = start;
+        this.size = size;
+
+        this.currentId = new AtomicLong(start);
+    }
+
+    // TODO: consider if this method is needed or not
+    /**
+     * Returns the initial value.
+     *
+     * @return initial value
+     */
+    public long getStart() {
+        return start;
+    }
+
+    // TODO: consider if this method is needed or not
+    /**
+     * Returns the last value.
+     *
+     * @return last value
+     */
+    public long getEnd() {
+        return start + size - 1;
+    }
+
+    /**
+     * Returns the block size.
+     *
+     * @return block size
+     */
+    public long getSize() {
+        return size;
+    }
+
+    /**
+     * Returns the next ID in the block.
+     *
+     * @return next ID
+     * @throws UnavailableIdException if there is no available ID in the block.
+     */
+    public long getNextId() {
+        final long id = currentId.getAndIncrement();
+        if (id > getEnd()) {
+            throw new UnavailableIdException(String.format(
+                    "used all IDs in allocated space (size: %d, end: %d, current: %d)",
+                    size, getEnd(), id
+            ));
+        }
+
+        return id;
+    }
+
+    // TODO: Do we really need equals and hashCode? Should it contain currentId?
+    @Override
+    public boolean equals(Object o) {
+        if (this == o) {
+            return true;
+        }
+        if (o == null || getClass() != o.getClass()) {
+            return false;
+        }
+
+        IdBlock that = (IdBlock) o;
+        return Objects.equals(this.start, that.start)
+                && Objects.equals(this.size, that.size)
+                && Objects.equals(this.currentId.get(), that.currentId.get());
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(start, size, currentId);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                .add("start", start)
+                .add("size", size)
+                .add("currentId", currentId)
+                .toString();
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+/**
+ * An interface that gives unique ID spaces.
+ */
+public interface IdBlockAllocator {
+    /**
+     * Allocates a unique Id Block.
+     *
+     * @return Id Block.
+     */
+    IdBlock allocateUniqueIdBlock();
+
+    /**
+     * Allocates next unique id and retrieve a new range of ids if needed.
+     *
+     * @param range range to use for the identifier
+     * @return Id Block.
+     */
+    IdBlock allocateUniqueIdBlock(long range);
+}

+package org.onlab.onos.net.intent.impl;
+
+import org.onlab.onos.net.intent.IntentId;
+
+/**
+ * An implementation of {@link org.onlab.onos.net.intent.IdGenerator} of intent ID,
+ * which uses {@link IdBlockAllocator}.
+ */
+public class IdBlockAllocatorBasedIntentIdGenerator extends AbstractBlockAllocatorBasedIdGenerator<IntentId> {
+
+    /**
+     * Constructs an intent ID generator, which uses the specified ID block allocator
+     * to generate a global unique intent ID.
+     *
+     * @param allocator the ID block allocator to use for generating intent IDs
+     */
+    public IdBlockAllocatorBasedIntentIdGenerator(IdBlockAllocator allocator) {
+        super(allocator);
+    }
+
+    @Override
+    protected IntentId convertFrom(long value) {
+        return new IntentId(value);
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+import org.onlab.onos.net.intent.IntentException;
+
+/**
+ * An exception thrown when a intent compilation fails.
+ */
+public class IntentCompilationException extends IntentException {
+    private static final long serialVersionUID = 235237603018210810L;
+
+    public IntentCompilationException() {
+        super();
+    }
+
+    public IntentCompilationException(String message) {
+        super(message);
+    }
+
+    public IntentCompilationException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+import org.onlab.onos.net.intent.IntentException;
+
+/**
+ * An exception thrown when intent installation fails.
+ */
+public class IntentInstallationException extends IntentException {
+    private static final long serialVersionUID = 3720268258616014168L;
+
+    public IntentInstallationException() {
+        super();
+    }
+
+    public IntentInstallationException(String message) {
+        super(message);
+    }
+
+    public IntentInstallationException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.onos.net.intent.IntentState.COMPILING;
+import static org.onlab.onos.net.intent.IntentState.FAILED;
+import static org.onlab.onos.net.intent.IntentState.INSTALLED;
+import static org.onlab.onos.net.intent.IntentState.INSTALLING;
+import static org.onlab.onos.net.intent.IntentState.RECOMPILING;
+import static org.onlab.onos.net.intent.IntentState.WITHDRAWING;
+import static org.onlab.onos.net.intent.IntentState.WITHDRAWN;
+import static org.onlab.util.Tools.namedThreads;
+import static org.slf4j.LoggerFactory.getLogger;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.ExecutorService;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.event.AbstractListenerRegistry;
+import org.onlab.onos.event.EventDeliveryService;
+import org.onlab.onos.net.intent.InstallableIntent;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentCompiler;
+import org.onlab.onos.net.intent.IntentEvent;
+import org.onlab.onos.net.intent.IntentException;
+import org.onlab.onos.net.intent.IntentExtensionService;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.IntentInstaller;
+import org.onlab.onos.net.intent.IntentListener;
+import org.onlab.onos.net.intent.IntentOperations;
+import org.onlab.onos.net.intent.IntentService;
+import org.onlab.onos.net.intent.IntentState;
+import org.onlab.onos.net.intent.IntentStore;
+import org.onlab.onos.net.intent.IntentStoreDelegate;
+import org.slf4j.Logger;
+
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * An implementation of Intent Manager.
+ */
+@Component(immediate = true)
+@Service
+public class IntentManager
+        implements IntentService, IntentExtensionService {
+    private final Logger log = getLogger(getClass());
+
+    public static final String INTENT_NULL = "Intent cannot be null";
+    public static final String INTENT_ID_NULL = "Intent ID cannot be null";
+
+    // Collections for compiler, installer, and listener are ONOS instance local
+    private final ConcurrentMap<Class<? extends Intent>,
+            IntentCompiler<? extends Intent>> compilers = new ConcurrentHashMap<>();
+    private final ConcurrentMap<Class<? extends InstallableIntent>,
+            IntentInstaller<? extends InstallableIntent>> installers = new ConcurrentHashMap<>();
+
+    private final AbstractListenerRegistry<IntentEvent, IntentListener>
+            listenerRegistry = new AbstractListenerRegistry<>();
+
+    private final ExecutorService executor = newSingleThreadExecutor(namedThreads("onos-intents"));
+
+    private final IntentStoreDelegate delegate = new InternalStoreDelegate();
+    private final TopologyChangeDelegate topoDelegate = new InternalTopoChangeDelegate();
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected IntentStore store;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ObjectiveTrackerService trackerService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected EventDeliveryService eventDispatcher;
+
+    @Activate
+    public void activate() {
+        store.setDelegate(delegate);
+        trackerService.setDelegate(topoDelegate);
+        eventDispatcher.addSink(IntentEvent.class, listenerRegistry);
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        store.unsetDelegate(delegate);
+        trackerService.unsetDelegate(topoDelegate);
+        eventDispatcher.removeSink(IntentEvent.class);
+        log.info("Stopped");
+    }
+
+    @Override
+    public void submit(Intent intent) {
+        checkNotNull(intent, INTENT_NULL);
+        registerSubclassCompilerIfNeeded(intent);
+        IntentEvent event = store.createIntent(intent);
+        if (event != null) {
+            eventDispatcher.post(event);
+            executor.execute(new IntentTask(COMPILING, intent));
+        }
+    }
+
+    @Override
+    public void withdraw(Intent intent) {
+        checkNotNull(intent, INTENT_NULL);
+        executor.execute(new IntentTask(WITHDRAWING, intent));
+    }
+
+    // FIXME: implement this method
+    @Override
+    public void execute(IntentOperations operations) {
+        throw new UnsupportedOperationException("execute() is not implemented yet");
+    }
+
+    @Override
+    public Iterable<Intent> getIntents() {
+        return store.getIntents();
+    }
+
+    @Override
+    public long getIntentCount() {
+        return store.getIntentCount();
+    }
+
+    @Override
+    public Intent getIntent(IntentId id) {
+        checkNotNull(id, INTENT_ID_NULL);
+        return store.getIntent(id);
+    }
+
+    @Override
+    public IntentState getIntentState(IntentId id) {
+        checkNotNull(id, INTENT_ID_NULL);
+        return store.getIntentState(id);
+    }
+
+    @Override
+    public void addListener(IntentListener listener) {
+        listenerRegistry.addListener(listener);
+    }
+
+    @Override
+    public void removeListener(IntentListener listener) {
+        listenerRegistry.removeListener(listener);
+    }
+
+    @Override
+    public <T extends Intent> void registerCompiler(Class<T> cls, IntentCompiler<T> compiler) {
+        compilers.put(cls, compiler);
+    }
+
+    @Override
+    public <T extends Intent> void unregisterCompiler(Class<T> cls) {
+        compilers.remove(cls);
+    }
+
+    @Override
+    public Map<Class<? extends Intent>, IntentCompiler<? extends Intent>> getCompilers() {
+        return ImmutableMap.copyOf(compilers);
+    }
+
+    @Override
+    public <T extends InstallableIntent> void registerInstaller(Class<T> cls, IntentInstaller<T> installer) {
+        installers.put(cls, installer);
+    }
+
+    @Override
+    public <T extends InstallableIntent> void unregisterInstaller(Class<T> cls) {
+        installers.remove(cls);
+    }
+
+    @Override
+    public Map<Class<? extends InstallableIntent>, IntentInstaller<? extends InstallableIntent>> getInstallers() {
+        return ImmutableMap.copyOf(installers);
+    }
+
+    /**
+     * Returns the corresponding intent compiler to the specified intent.
+     *
+     * @param intent intent
+     * @param <T>    the type of intent
+     * @return intent compiler corresponding to the specified intent
+     */
+    private <T extends Intent> IntentCompiler<T> getCompiler(T intent) {
+        @SuppressWarnings("unchecked")
+        IntentCompiler<T> compiler = (IntentCompiler<T>) compilers.get(intent.getClass());
+        if (compiler == null) {
+            throw new IntentException("no compiler for class " + intent.getClass());
+        }
+        return compiler;
+    }
+
+    /**
+     * Returns the corresponding intent installer to the specified installable intent.
+     *
+     * @param intent intent
+     * @param <T>    the type of installable intent
+     * @return intent installer corresponding to the specified installable intent
+     */
+    private <T extends InstallableIntent> IntentInstaller<T> getInstaller(T intent) {
+        @SuppressWarnings("unchecked")
+        IntentInstaller<T> installer = (IntentInstaller<T>) installers.get(intent.getClass());
+        if (installer == null) {
+            throw new IntentException("no installer for class " + intent.getClass());
+        }
+        return installer;
+    }
+
+    /**
+     * Compiles the specified intent.
+     *
+     * @param intent intent to be compiled
+     */
+    private void executeCompilingPhase(Intent intent) {
+        // Indicate that the intent is entering the compiling phase.
+        store.setState(intent, COMPILING);
+
+        try {
+            // Compile the intent into installable derivatives.
+            List<InstallableIntent> installable = compileIntent(intent);
+
+            // If all went well, associate the resulting list of installable
+            // intents with the top-level intent and proceed to install.
+            store.addInstallableIntents(intent.id(), installable);
+            executeInstallingPhase(intent);
+
+        } catch (Exception e) {
+            log.warn("Unable to compile intent {} due to: {}", intent.id(), e);
+
+            // If compilation failed, mark the intent as failed.
+            store.setState(intent, FAILED);
+        }
+    }
+
+    // FIXME: To make SDN-IP workable ASAP, only single level compilation is implemented
+    // TODO: implement compilation traversing tree structure
+    private List<InstallableIntent> compileIntent(Intent intent) {
+        List<InstallableIntent> installable = new ArrayList<>();
+        for (Intent compiled : getCompiler(intent).compile(intent)) {
+            InstallableIntent installableIntent = (InstallableIntent) compiled;
+            installable.add(installableIntent);
+        }
+        return installable;
+    }
+
+    /**
+     * Installs all installable intents associated with the specified top-level
+     * intent.
+     *
+     * @param intent intent to be installed
+     */
+    private void executeInstallingPhase(Intent intent) {
+        // Indicate that the intent is entering the installing phase.
+        store.setState(intent, INSTALLING);
+
+        try {
+            List<InstallableIntent> installables = store.getInstallableIntents(intent.id());
+            if (installables != null) {
+                for (InstallableIntent installable : installables) {
+                    registerSubclassInstallerIfNeeded(installable);
+                    trackerService.addTrackedResources(intent.id(),
+                                                       installable.requiredLinks());
+                    getInstaller(installable).install(installable);
+                }
+            }
+            eventDispatcher.post(store.setState(intent, INSTALLED));
+
+        } catch (Exception e) {
+            log.warn("Unable to install intent {} due to: {}", intent.id(), e);
+            uninstallIntent(intent);
+
+            // If compilation failed, kick off the recompiling phase.
+            executeRecompilingPhase(intent);
+        }
+    }
+
+    /**
+     * Recompiles the specified intent.
+     *
+     * @param intent intent to be recompiled
+     */
+    private void executeRecompilingPhase(Intent intent) {
+        // Indicate that the intent is entering the recompiling phase.
+        store.setState(intent, RECOMPILING);
+
+        try {
+            // Compile the intent into installable derivatives.
+            List<InstallableIntent> installable = compileIntent(intent);
+
+            // If all went well, compare the existing list of installable
+            // intents with the newly compiled list. If they are the same,
+            // bail, out since the previous approach was determined not to
+            // be viable.
+            List<InstallableIntent> originalInstallable =
+                    store.getInstallableIntents(intent.id());
+
+            if (Objects.equals(originalInstallable, installable)) {
+                eventDispatcher.post(store.setState(intent, FAILED));
+            } else {
+                // Otherwise, re-associate the newly compiled installable intents
+                // with the top-level intent and kick off installing phase.
+                store.addInstallableIntents(intent.id(), installable);
+                executeInstallingPhase(intent);
+            }
+        } catch (Exception e) {
+            log.warn("Unable to recompile intent {} due to: {}", intent.id(), e);
+
+            // If compilation failed, mark the intent as failed.
+            eventDispatcher.post(store.setState(intent, FAILED));
+        }
+    }
+
+    /**
+     * Uninstalls the specified intent by uninstalling all of its associated
+     * installable derivatives.
+     *
+     * @param intent intent to be installed
+     */
+    private void executeWithdrawingPhase(Intent intent) {
+        // Indicate that the intent is being withdrawn.
+        store.setState(intent, WITHDRAWING);
+        uninstallIntent(intent);
+
+        // If all went well, disassociate the top-level intent with its
+        // installable derivatives and mark it as withdrawn.
+        store.removeInstalledIntents(intent.id());
+        eventDispatcher.post(store.setState(intent, WITHDRAWN));
+    }
+
+    /**
+     * Uninstalls all installable intents associated with the given intent.
+     *
+     * @param intent intent to be uninstalled
+     */
+    private void uninstallIntent(Intent intent) {
+        try {
+            List<InstallableIntent> installables = store.getInstallableIntents(intent.id());
+            if (installables != null) {
+                for (InstallableIntent installable : installables) {
+                    getInstaller(installable).uninstall(installable);
+                }
+            }
+        } catch (IntentException e) {
+            log.warn("Unable to uninstall intent {} due to: {}", intent.id(), e);
+        }
+    }
+
+    /**
+     * Registers an intent compiler of the specified intent if an intent compiler
+     * for the intent is not registered. This method traverses the class hierarchy of
+     * the intent. Once an intent compiler for a parent type is found, this method
+     * registers the found intent compiler.
+     *
+     * @param intent intent
+     */
+    private void registerSubclassCompilerIfNeeded(Intent intent) {
+        if (!compilers.containsKey(intent.getClass())) {
+            Class<?> cls = intent.getClass();
+            while (cls != Object.class) {
+                // As long as we're within the Intent class descendants
+                if (Intent.class.isAssignableFrom(cls)) {
+                    IntentCompiler<?> compiler = compilers.get(cls);
+                    if (compiler != null) {
+                        compilers.put(intent.getClass(), compiler);
+                        return;
+                    }
+                }
+                cls = cls.getSuperclass();
+            }
+        }
+    }
+
+    /**
+     * Registers an intent installer of the specified intent if an intent installer
+     * for the intent is not registered. This method traverses the class hierarchy of
+     * the intent. Once an intent installer for a parent type is found, this method
+     * registers the found intent installer.
+     *
+     * @param intent intent
+     */
+    private void registerSubclassInstallerIfNeeded(InstallableIntent intent) {
+        if (!installers.containsKey(intent.getClass())) {
+            Class<?> cls = intent.getClass();
+            while (cls != Object.class) {
+                // As long as we're within the InstallableIntent class descendants
+                if (InstallableIntent.class.isAssignableFrom(cls)) {
+                    IntentInstaller<?> installer = installers.get(cls);
+                    if (installer != null) {
+                        installers.put(intent.getClass(), installer);
+                        return;
+                    }
+                }
+                cls = cls.getSuperclass();
+            }
+        }
+    }
+
+    // Store delegate to re-post events emitted from the store.
+    private class InternalStoreDelegate implements IntentStoreDelegate {
+        @Override
+        public void notify(IntentEvent event) {
+            eventDispatcher.post(event);
+            if (event.type() == IntentEvent.Type.SUBMITTED) {
+                executor.execute(new IntentTask(COMPILING, event.subject()));
+            }
+        }
+    }
+
+    // Topology change delegate
+    private class InternalTopoChangeDelegate implements TopologyChangeDelegate {
+        @Override
+        public void triggerCompile(Iterable<IntentId> intentIds,
+                                   boolean compileAllFailed) {
+            // Attempt recompilation of the specified intents first.
+            for (IntentId intentId : intentIds) {
+                Intent intent = getIntent(intentId);
+                uninstallIntent(intent);
+
+                executeRecompilingPhase(intent);
+            }
+
+            if (compileAllFailed) {
+                // If required, compile all currently failed intents.
+                for (Intent intent : getIntents()) {
+                    if (getIntentState(intent.id()) == FAILED) {
+                        executeCompilingPhase(intent);
+                    }
+                }
+            }
+        }
+    }
+
+    // Auxiliary runnable to perform asynchronous tasks.
+    private class IntentTask implements Runnable {
+        private final IntentState state;
+        private final Intent intent;
+
+        public IntentTask(IntentState state, Intent intent) {
+            this.state = state;
+            this.intent = intent;
+        }
+
+        @Override
+        public void run() {
+            if (state == COMPILING) {
+                executeCompilingPhase(intent);
+            } else if (state == RECOMPILING) {
+                executeRecompilingPhase(intent);
+            } else if (state == WITHDRAWING) {
+                executeWithdrawingPhase(intent);
+            }
+        }
+    }
+
+}

+package org.onlab.onos.net.intent.impl;
+
+import org.onlab.onos.net.intent.IntentException;
+
+/**
+ * An exception thrown when intent removal failed.
+ */
+public class IntentRemovalException extends IntentException {
+    private static final long serialVersionUID = -5259226322037891951L;
+
+    public IntentRemovalException() {
+        super();
+    }
+
+    public IntentRemovalException(String message) {
+        super(message);
+    }
+
+    public IntentRemovalException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.SetMultimap;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.event.Event;
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.LinkKey;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.link.LinkEvent;
+import org.onlab.onos.net.topology.TopologyEvent;
+import org.onlab.onos.net.topology.TopologyListener;
+import org.onlab.onos.net.topology.TopologyService;
+import org.slf4j.Logger;
+
+import java.util.Collection;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.collect.Multimaps.synchronizedSetMultimap;
+import static java.util.concurrent.Executors.newSingleThreadExecutor;
+import static org.onlab.onos.net.link.LinkEvent.Type.LINK_REMOVED;
+import static org.onlab.util.Tools.namedThreads;
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Entity responsible for tracking installed flows and for monitoring topology
+ * events to determine what flows are affected by topology changes.
+ */
+@Component
+@Service
+public class ObjectiveTracker implements ObjectiveTrackerService {
+
+    private final Logger log = getLogger(getClass());
+
+    private final SetMultimap<LinkKey, IntentId> intentsByLink =
+            synchronizedSetMultimap(HashMultimap.<LinkKey, IntentId>create());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected TopologyService topologyService;
+
+    private ExecutorService executorService =
+            newSingleThreadExecutor(namedThreads("onos-flowtracker"));
+
+    private TopologyListener listener = new InternalTopologyListener();
+    private TopologyChangeDelegate delegate;
+
+    @Activate
+    public void activate() {
+        topologyService.addListener(listener);
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        topologyService.removeListener(listener);
+        log.info("Stopped");
+    }
+
+    @Override
+    public void setDelegate(TopologyChangeDelegate delegate) {
+        checkNotNull(delegate, "Delegate cannot be null");
+        checkArgument(this.delegate == null || this.delegate == delegate,
+                      "Another delegate already set");
+        this.delegate = delegate;
+    }
+
+    @Override
+    public void unsetDelegate(TopologyChangeDelegate delegate) {
+        checkArgument(this.delegate == delegate, "Not the current delegate");
+        this.delegate = null;
+    }
+
+    @Override
+    public void addTrackedResources(IntentId intentId, Collection<Link> resources) {
+        for (Link link : resources) {
+            intentsByLink.put(new LinkKey(link), intentId);
+        }
+    }
+
+    @Override
+    public void removeTrackedResources(IntentId intentId, Collection<Link> resources) {
+        for (Link link : resources) {
+            intentsByLink.remove(new LinkKey(link), intentId);
+        }
+    }
+
+    // Internal re-actor to topology change events.
+    private class InternalTopologyListener implements TopologyListener {
+        @Override
+        public void event(TopologyEvent event) {
+            executorService.execute(new TopologyChangeHandler(event));
+        }
+    }
+
+    // Re-dispatcher of topology change events.
+    private class TopologyChangeHandler implements Runnable {
+
+        private final TopologyEvent event;
+
+        TopologyChangeHandler(TopologyEvent event) {
+            this.event = event;
+        }
+
+        @Override
+        public void run() {
+            if (event.reasons() == null) {
+                delegate.triggerCompile(new HashSet<IntentId>(), true);
+
+            } else {
+                Set<IntentId> toBeRecompiled = new HashSet<>();
+                boolean recompileOnly = true;
+
+                // Scan through the list of reasons and keep accruing all
+                // intents that need to be recompiled.
+                for (Event reason : event.reasons()) {
+                    if (reason instanceof LinkEvent) {
+                        LinkEvent linkEvent = (LinkEvent) reason;
+                        if (linkEvent.type() == LINK_REMOVED) {
+                            Set<IntentId> intentIds = intentsByLink.get(new LinkKey(linkEvent.subject()));
+                            toBeRecompiled.addAll(intentIds);
+                        }
+                        recompileOnly = recompileOnly && linkEvent.type() == LINK_REMOVED;
+                    }
+                }
+
+                delegate.triggerCompile(toBeRecompiled, !recompileOnly);
+            }
+        }
+    }
+
+}

+package org.onlab.onos.net.intent.impl;
+
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.intent.IntentId;
+
+import java.util.Collection;
+
+/**
+ * Auxiliary service for tracking intent path flows and for notifying the
+ * intent service of environment changes via topology change delegate.
+ */
+public interface ObjectiveTrackerService {
+
+    /**
+     * Sets a topology change delegate.
+     *
+     * @param delegate topology change delegate
+     */
+    void setDelegate(TopologyChangeDelegate delegate);
+
+    /**
+     * Unsets topology change delegate.
+     *
+     * @param delegate topology change delegate
+     */
+    void unsetDelegate(TopologyChangeDelegate delegate);
+
+    /**
+     * Adds a path flow to be tracked.
+     *
+     * @param intentId intent identity on whose behalf the path is being tracked
+     * @param resources resources to track
+     */
+    public void addTrackedResources(IntentId intentId, Collection<Link> resources);
+
+    /**
+     * Removes a path flow to be tracked.
+     *
+     * @param intentId intent identity on whose behalf the path is being tracked
+     * @param resources resources to stop tracking
+     */
+    public void removeTrackedResources(IntentId intentId, Collection<Link> resources);
+
+}

+package org.onlab.onos.net.intent.impl;
+
+import static org.onlab.onos.net.flow.DefaultTrafficTreatment.builder;
+import static org.slf4j.LoggerFactory.getLogger;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.onlab.onos.ApplicationId;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.flow.DefaultFlowRule;
+import org.onlab.onos.net.flow.DefaultTrafficSelector;
+import org.onlab.onos.net.flow.FlowRule;
+import org.onlab.onos.net.flow.FlowRuleBatchEntry;
+import org.onlab.onos.net.flow.FlowRuleBatchEntry.FlowRuleOperation;
+import org.onlab.onos.net.flow.FlowRuleBatchOperation;
+import org.onlab.onos.net.flow.FlowRuleService;
+import org.onlab.onos.net.flow.TrafficSelector;
+import org.onlab.onos.net.flow.TrafficTreatment;
+import org.onlab.onos.net.intent.IntentExtensionService;
+import org.onlab.onos.net.intent.IntentInstaller;
+import org.onlab.onos.net.intent.PathIntent;
+import org.slf4j.Logger;
+
+import com.google.common.collect.Lists;
+
+/**
+ * Installer for {@link PathIntent path connectivity intents}.
+ */
+@Component(immediate = true)
+public class PathIntentInstaller implements IntentInstaller<PathIntent> {
+
+    private final Logger log = getLogger(getClass());
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected IntentExtensionService intentManager;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected FlowRuleService flowRuleService;
+
+    private final ApplicationId appId = ApplicationId.getAppId();
+
+    @Activate
+    public void activate() {
+        intentManager.registerInstaller(PathIntent.class, this);
+    }
+
+    @Deactivate
+    public void deactivate() {
+        intentManager.unregisterInstaller(PathIntent.class);
+    }
+
+    @Override
+    public void install(PathIntent intent) {
+        TrafficSelector.Builder builder =
+                DefaultTrafficSelector.builder(intent.selector());
+        Iterator<Link> links = intent.path().links().iterator();
+        ConnectPoint prev = links.next().dst();
+        List<FlowRuleBatchEntry> rules = Lists.newLinkedList();
+        while (links.hasNext()) {
+            builder.matchInport(prev.port());
+            Link link = links.next();
+            TrafficTreatment treatment = builder()
+                    .setOutput(link.src().port()).build();
+
+            FlowRule rule = new DefaultFlowRule(link.src().deviceId(),
+                    builder.build(), treatment,
+                    123, appId, 600);
+            rules.add(new FlowRuleBatchEntry(FlowRuleOperation.ADD, rule));
+            //flowRuleService.applyFlowRules(rule);
+            prev = link.dst();
+        }
+        FlowRuleBatchOperation batch = new FlowRuleBatchOperation(rules);
+        try {
+            flowRuleService.applyBatch(batch).get();
+        } catch (InterruptedException | ExecutionException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public void uninstall(PathIntent intent) {
+        TrafficSelector.Builder builder =
+                DefaultTrafficSelector.builder(intent.selector());
+        Iterator<Link> links = intent.path().links().iterator();
+        ConnectPoint prev = links.next().dst();
+        List<FlowRuleBatchEntry> rules = Lists.newLinkedList();
+
+        while (links.hasNext()) {
+            builder.matchInport(prev.port());
+            Link link = links.next();
+            TrafficTreatment treatment = builder()
+                    .setOutput(link.src().port()).build();
+            FlowRule rule = new DefaultFlowRule(link.src().deviceId(),
+                    builder.build(), treatment,
+                    123, appId, 600);
+            rules.add(new FlowRuleBatchEntry(FlowRuleOperation.REMOVE, rule));
+            //flowRuleService.removeFlowRules(rule);
+            prev = link.dst();
+        }
+        FlowRuleBatchOperation batch = new FlowRuleBatchOperation(rules);
+        try {
+            flowRuleService.applyBatch(batch).get();
+        } catch (InterruptedException | ExecutionException e) {
+            // TODO Auto-generated catch block
+            e.printStackTrace();
+        }
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+import org.onlab.onos.net.intent.IntentException;
+
+/**
+ * An exception thrown when a path is not found.
+ */
+public class PathNotFoundException extends IntentException {
+    private static final long serialVersionUID = -2087045731049914733L;
+
+    public PathNotFoundException() {
+        super();
+    }
+
+    public PathNotFoundException(String message) {
+        super(message);
+    }
+
+    public PathNotFoundException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DefaultEdgeLink;
+import org.onlab.onos.net.DefaultPath;
+import org.onlab.onos.net.Link;
+import org.onlab.onos.net.Path;
+import org.onlab.onos.net.host.HostService;
+import org.onlab.onos.net.intent.IdGenerator;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentCompiler;
+import org.onlab.onos.net.intent.IntentExtensionService;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.PathIntent;
+import org.onlab.onos.net.intent.PointToPointIntent;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.net.topology.PathService;
+
+/**
+ * A intent compiler for {@link org.onlab.onos.net.intent.HostToHostIntent}.
+ */
+@Component(immediate = true)
+public class PointToPointIntentCompiler
+        implements IntentCompiler<PointToPointIntent> {
+
+    private static final ProviderId PID = new ProviderId("core", "org.onlab.onos.core", true);
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected IntentExtensionService intentManager;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected PathService pathService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected HostService hostService;
+
+    private IdGenerator<IntentId> intentIdGenerator;
+
+    @Activate
+    public void activate() {
+        IdBlockAllocator idBlockAllocator = new DummyIdBlockAllocator();
+        intentIdGenerator = new IdBlockAllocatorBasedIntentIdGenerator(idBlockAllocator);
+        intentManager.registerCompiler(PointToPointIntent.class, this);
+    }
+
+    @Deactivate
+    public void deactivate() {
+        intentManager.unregisterCompiler(PointToPointIntent.class);
+    }
+
+    @Override
+    public List<Intent> compile(PointToPointIntent intent) {
+        Path path = getPath(intent.ingressPoint(), intent.egressPoint());
+
+        List<Link> links = new ArrayList<>();
+        links.add(DefaultEdgeLink.createEdgeLink(intent.ingressPoint(), true));
+        links.addAll(path.links());
+        links.add(DefaultEdgeLink.createEdgeLink(intent.egressPoint(), false));
+
+        return Arrays.asList(createPathIntent(new DefaultPath(PID, links, path.cost() + 2,
+                                              path.annotations()),
+                             intent));
+    }
+
+    /**
+     * Creates a path intent from the specified path and original
+     * connectivity intent.
+     *
+     * @param path path to create an intent for
+     * @param intent original intent
+     */
+    private Intent createPathIntent(Path path,
+                                    PointToPointIntent intent) {
+
+        return new PathIntent(intentIdGenerator.getNewId(),
+                              intent.selector(), intent.treatment(),
+                              path.src(), path.dst(), path);
+    }
+
+    /**
+     * Computes a path between two ConnectPoints.
+     *
+     * @param one start of the path
+     * @param two end of the path
+     * @return Path between the two
+     * @throws PathNotFoundException if a path cannot be found
+     */
+    private Path getPath(ConnectPoint one, ConnectPoint two) {
+        Set<Path> paths = pathService.getPaths(one.deviceId(), two.deviceId());
+        if (paths.isEmpty()) {
+            throw new PathNotFoundException("No path from " + one + " to " + two);
+        }
+        // TODO: let's be more intelligent about this eventually
+        return paths.iterator().next();
+    }
+}

+package org.onlab.onos.net.intent.impl;
+
+import org.onlab.onos.net.intent.IntentId;
+
+/**
+ * Auxiliary delegate for integration of intent manager and flow trackerService.
+ */
+public interface TopologyChangeDelegate {
+
+    /**
+     * Notifies that topology has changed in such a way that the specified
+     * intents should be recompiled. If the {@code compileAllFailed} parameter
+     * is true, then all intents in {@link org.onlab.onos.net.intent.IntentState#FAILED}
+     * state should be compiled as well.
+     *
+     * @param intentIds intents that should be recompiled
+     * @param compileAllFailed true implies full compile of all failed intents
+     *                         is required; false for selective recompile only
+     */
+    void triggerCompile(Iterable<IntentId> intentIds, boolean compileAllFailed);
+
+}

+package org.onlab.onos.net.intent.impl;
+
+/**
+ * Represents that there is no available IDs.
+ */
+public class UnavailableIdException extends RuntimeException {
+
+    private static final long serialVersionUID = -2287403908433720122L;
+
+    /**
+     * Constructs an exception with no message and no underlying cause.
+     */
+    public UnavailableIdException() {
+    }
+
+    /**
+     * Constructs an exception with the specified message.
+     *
+     * @param message the message describing the specific nature of the error
+     */
+    public UnavailableIdException(String message) {
+        super(message);
+    }
+
+    /**
+     * Constructs an exception with the specified message and the underlying cause.
+     *
+     * @param message the message describing the specific nature of the error
+     * @param cause the underlying cause of this error
+     */
+    public UnavailableIdException(String message, Throwable cause) {
+        super(message, cause);
+    }
+}

+/**
+ * Core subsystem for tracking high-level intents for treatment of selected
+ * network traffic.
+ */
+package org.onlab.onos.net.intent.impl;
\ No newline at end of file

@@ -31,6 +31,8 @@ import org.onlab.onos.net.link.LinkEvent;
 import org.onlab.onos.net.link.LinkListener;
 import org.onlab.onos.net.link.LinkService;
 import org.onlab.onos.net.packet.DefaultOutboundPacket;
+import org.onlab.onos.net.packet.InboundPacket;
+import org.onlab.onos.net.packet.PacketContext;
 import org.onlab.onos.net.packet.PacketService;
 import org.onlab.onos.net.proxyarp.ProxyArpService;
 import org.onlab.packet.ARP;
@@ -43,7 +45,6 @@ import com.google.common.collect.HashMultimap;
 import com.google.common.collect.Lists;
 import com.google.common.collect.Multimap;
 
-
 @Component(immediate = true)
 @Service
 public class ProxyArpManager implements ProxyArpService {
@@ -93,14 +94,14 @@ public class ProxyArpManager implements ProxyArpService {
 
     @Override
     public boolean known(IpPrefix addr) {
-        checkNotNull(MAC_ADDR_NULL, addr);
+        checkNotNull(addr, MAC_ADDR_NULL);
         Set<Host> hosts = hostService.getHostsByIp(addr);
         return !hosts.isEmpty();
     }
 
     @Override
     public void reply(Ethernet eth) {
-        checkNotNull(REQUEST_NULL, eth);
+        checkNotNull(eth, REQUEST_NULL);
         checkArgument(eth.getEtherType() == Ethernet.TYPE_ARP,
                 REQUEST_NOT_ARP);
         ARP arp = (ARP) eth.getPayload();
@@ -128,7 +129,7 @@ public class ProxyArpManager implements ProxyArpService {
 
         Ethernet arpReply = buildArpReply(dst, eth);
         // TODO: check send status with host service.
-        TrafficTreatment.Builder builder = new DefaultTrafficTreatment.Builder();
+        TrafficTreatment.Builder builder = DefaultTrafficTreatment.builder();
         builder.setOutput(src.location().port());
         packetService.emit(new DefaultOutboundPacket(src.location().deviceId(),
                 builder.build(), ByteBuffer.wrap(arpReply.serialize())));
@@ -136,7 +137,7 @@ public class ProxyArpManager implements ProxyArpService {
 
     @Override
     public void forward(Ethernet eth) {
-        checkNotNull(REQUEST_NULL, eth);
+        checkNotNull(eth, REQUEST_NULL);
         checkArgument(eth.getEtherType() == Ethernet.TYPE_ARP,
                 REQUEST_NOT_ARP);
         ARP arp = (ARP) eth.getPayload();
@@ -148,7 +149,7 @@ public class ProxyArpManager implements ProxyArpService {
         if (h == null) {
             flood(eth);
         } else {
-            TrafficTreatment.Builder builder = new DefaultTrafficTreatment.Builder();
+            TrafficTreatment.Builder builder = DefaultTrafficTreatment.builder();
             builder.setOutput(h.location().port());
             packetService.emit(new DefaultOutboundPacket(h.location().deviceId(),
                     builder.build(), ByteBuffer.wrap(eth.serialize())));
@@ -156,6 +157,23 @@ public class ProxyArpManager implements ProxyArpService {
 
     }
 
+    @Override
+    public boolean handleArp(PacketContext context) {
+        InboundPacket pkt = context.inPacket();
+        Ethernet ethPkt = pkt.parsed();
+        if (ethPkt.getEtherType() == Ethernet.TYPE_ARP) {
+            ARP arp = (ARP) ethPkt.getPayload();
+            if (arp.getOpCode() == ARP.OP_REPLY) {
+                forward(ethPkt);
+            } else if (arp.getOpCode() == ARP.OP_REQUEST) {
+                reply(ethPkt);
+            }
+            context.block();
+            return true;
+        }
+        return false;
+    }
+
     /**
      * Flood the arp request at all edges in the network.
      * @param request the arp request.
@@ -166,7 +184,7 @@ public class ProxyArpManager implements ProxyArpService {
 
         synchronized (externalPorts) {
             for (Entry<Device, PortNumber> entry : externalPorts.entries()) {
-                builder = new DefaultTrafficTreatment.Builder();
+                builder = DefaultTrafficTreatment.builder();
                 builder.setOutput(entry.getValue());
                 packetService.emit(new DefaultOutboundPacket(entry.getKey().id(),
                         builder.build(), buf));
@@ -188,12 +206,12 @@ public class ProxyArpManager implements ProxyArpService {
             for (Link l : links) {
                 // for each link, mark the concerned ports as internal
                 // and the remaining ports are therefore external.
-                if (l.src().deviceId().equals(d)
+                if (l.src().deviceId().equals(d.id())
                         && ports.contains(l.src().port())) {
                     ports.remove(l.src().port());
                     internalPorts.put(d, l.src().port());
                 }
-                if (l.dst().deviceId().equals(d)
+                if (l.dst().deviceId().equals(d.id())
                         && ports.contains(l.dst().port())) {
                     ports.remove(l.dst().port());
                     internalPorts.put(d, l.dst().port());
@@ -322,7 +340,6 @@ public class ProxyArpManager implements ProxyArpService {
 
         }
 
-}
-
+    }
 
 }

@@ -6,8 +6,15 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Ignore;
 import org.junit.Test;
+import org.onlab.onos.cluster.ClusterEventListener;
+import org.onlab.onos.cluster.ClusterService;
+import org.onlab.onos.cluster.ControllerNode;
+import org.onlab.onos.cluster.DefaultControllerNode;
 import org.onlab.onos.cluster.MastershipServiceAdapter;
+import org.onlab.onos.cluster.MastershipTerm;
+import org.onlab.onos.cluster.MastershipTermService;
 import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.cluster.ControllerNode.State;
 import org.onlab.onos.event.Event;
 import org.onlab.onos.event.impl.TestEventDispatcher;
 import org.onlab.onos.net.Device;
@@ -28,7 +35,9 @@ import org.onlab.onos.net.device.DeviceService;
 import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.AbstractProvider;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.ClockProviderService;
 import org.onlab.onos.store.trivial.impl.SimpleDeviceStore;
+import org.onlab.packet.IpPrefix;
 
 import java.util.ArrayList;
 import java.util.Iterator;
@@ -57,6 +66,8 @@ public class DeviceManagerTest {
     private static final PortNumber P1 = PortNumber.portNumber(1);
     private static final PortNumber P2 = PortNumber.portNumber(2);
     private static final PortNumber P3 = PortNumber.portNumber(3);
+    private static final NodeId NID_LOCAL = new NodeId("local");
+    private static final IpPrefix LOCALHOST = IpPrefix.valueOf("127.0.0.1");
 
     private DeviceManager mgr;
 
@@ -76,6 +87,8 @@ public class DeviceManagerTest {
         mgr.store = new SimpleDeviceStore();
         mgr.eventDispatcher = new TestEventDispatcher();
         mgr.mastershipService = new TestMastershipService();
+        mgr.clusterService = new TestClusterService();
+        mgr.clockProviderService = new TestClockProviderService();
         mgr.activate();
 
         service.addListener(listener);
@@ -275,6 +288,59 @@ public class DeviceManagerTest {
         public MastershipRole requestRoleFor(DeviceId deviceId) {
             return MastershipRole.MASTER;
         }
+
+        @Override
+        public MastershipTermService requestTermService() {
+            return new MastershipTermService() {
+                @Override
+                public MastershipTerm getMastershipTerm(DeviceId deviceId) {
+                    // FIXME: just returning something not null
+                    return MastershipTerm.of(NID_LOCAL, 1);
+                }
+            };
+        }
+    }
+
+    // code clone
+    private final class TestClusterService implements ClusterService {
+
+        ControllerNode local = new DefaultControllerNode(NID_LOCAL, LOCALHOST);
+
+        @Override
+        public ControllerNode getLocalNode() {
+            return local;
+        }
+
+        @Override
+        public Set<ControllerNode> getNodes() {
+            return null;
+        }
+
+        @Override
+        public ControllerNode getNode(NodeId nodeId) {
+            return null;
+        }
+
+        @Override
+        public State getState(NodeId nodeId) {
+            return null;
+        }
+
+        @Override
+        public void addListener(ClusterEventListener listener) {
+        }
+
+        @Override
+        public void removeListener(ClusterEventListener listener) {
+        }
     }
 
+    private final class TestClockProviderService implements
+            ClockProviderService {
+
+        @Override
+        public void setMastershipTerm(DeviceId deviceId, MastershipTerm term) {
+            // TODO Auto-generated method stub
+        }
+    }
 }

-package org.onlab.onos.net.device.impl;
-
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Sets;
-import com.hazelcast.config.Config;
-import com.hazelcast.core.Hazelcast;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.onlab.onos.cluster.DefaultControllerNode;
-import org.onlab.onos.cluster.MastershipServiceAdapter;
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.event.Event;
-import org.onlab.onos.event.EventDeliveryService;
-import org.onlab.onos.event.impl.TestEventDispatcher;
-import org.onlab.onos.net.Device;
-import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.net.MastershipRole;
-import org.onlab.onos.net.Port;
-import org.onlab.onos.net.PortNumber;
-import org.onlab.onos.net.device.DefaultDeviceDescription;
-import org.onlab.onos.net.device.DefaultPortDescription;
-import org.onlab.onos.net.device.DeviceAdminService;
-import org.onlab.onos.net.device.DeviceDescription;
-import org.onlab.onos.net.device.DeviceEvent;
-import org.onlab.onos.net.device.DeviceListener;
-import org.onlab.onos.net.device.DeviceProvider;
-import org.onlab.onos.net.device.DeviceProviderRegistry;
-import org.onlab.onos.net.device.DeviceProviderService;
-import org.onlab.onos.net.device.DeviceService;
-import org.onlab.onos.net.device.PortDescription;
-import org.onlab.onos.net.provider.AbstractProvider;
-import org.onlab.onos.net.provider.ProviderId;
-import org.onlab.onos.store.common.StoreManager;
-import org.onlab.onos.store.common.StoreService;
-import org.onlab.onos.store.common.TestStoreManager;
-import org.onlab.onos.store.device.impl.DistributedDeviceStore;
-import org.onlab.onos.store.serializers.KryoSerializationManager;
-import org.onlab.onos.store.serializers.KryoSerializationService;
-import org.onlab.packet.IpPrefix;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
-
-import static org.junit.Assert.*;
-import static org.onlab.onos.net.Device.Type.SWITCH;
-import static org.onlab.onos.net.DeviceId.deviceId;
-import static org.onlab.onos.net.device.DeviceEvent.Type.*;
-
-// FIXME This test is slow starting up Hazelcast on each test cases.
-// FIXME DistributedDeviceStore should have it's own test cases.
-
-/**
- * Test codifying the device service & device provider service contracts.
- */
-public class DistributedDeviceManagerTest {
-
-    private static final ProviderId PID = new ProviderId("of", "foo");
-    private static final DeviceId DID1 = deviceId("of:foo");
-    private static final DeviceId DID2 = deviceId("of:bar");
-    private static final String MFR = "whitebox";
-    private static final String HW = "1.1.x";
-    private static final String SW1 = "3.8.1";
-    private static final String SW2 = "3.9.5";
-    private static final String SN = "43311-12345";
-
-    private static final PortNumber P1 = PortNumber.portNumber(1);
-    private static final PortNumber P2 = PortNumber.portNumber(2);
-    private static final PortNumber P3 = PortNumber.portNumber(3);
-
-    private static final DefaultControllerNode SELF
-        = new DefaultControllerNode(new NodeId("foobar"),
-                        IpPrefix.valueOf("127.0.0.1"));
-
-
-    private DeviceManager mgr;
-
-    protected StoreManager storeManager;
-    protected DeviceService service;
-    protected DeviceAdminService admin;
-    protected DeviceProviderRegistry registry;
-    protected DeviceProviderService providerService;
-    protected TestProvider provider;
-    protected TestListener listener = new TestListener();
-    private DistributedDeviceStore dstore;
-    private TestMastershipManager masterManager;
-    private EventDeliveryService eventService;
-    private KryoSerializationManager serializationMgr;
-
-    @Before
-    public void setUp() {
-        mgr = new DeviceManager();
-        service = mgr;
-        admin = mgr;
-        registry = mgr;
-        // TODO should find a way to clean Hazelcast instance without shutdown.
-        Config config = TestStoreManager.getTestConfig();
-
-        masterManager = new TestMastershipManager();
-
-        storeManager = new TestStoreManager(Hazelcast.newHazelcastInstance(config));
-        storeManager.activate();
-
-        serializationMgr = new KryoSerializationManager();
-        serializationMgr.activate();
-
-        dstore = new TestDistributedDeviceStore(storeManager, serializationMgr);
-        dstore.activate();
-
-        mgr.store = dstore;
-        eventService = new TestEventDispatcher();
-        mgr.eventDispatcher = eventService;
-        mgr.mastershipService = masterManager;
-        mgr.activate();
-
-        service.addListener(listener);
-
-        provider = new TestProvider();
-        providerService = registry.register(provider);
-        assertTrue("provider should be registered",
-                   registry.getProviders().contains(provider.id()));
-    }
-
-    @After
-    public void tearDown() {
-        registry.unregister(provider);
-        assertFalse("provider should not be registered",
-                    registry.getProviders().contains(provider.id()));
-        service.removeListener(listener);
-        mgr.deactivate();
-
-        dstore.deactivate();
-        serializationMgr.deactivate();
-        storeManager.deactivate();
-    }
-
-    private void connectDevice(DeviceId deviceId, String swVersion) {
-        DeviceDescription description =
-                new DefaultDeviceDescription(deviceId.uri(), SWITCH, MFR,
-                                             HW, swVersion, SN);
-        providerService.deviceConnected(deviceId, description);
-        assertNotNull("device should be found", service.getDevice(DID1));
-    }
-
-    @Test
-    public void deviceConnected() {
-        assertNull("device should not be found", service.getDevice(DID1));
-        connectDevice(DID1, SW1);
-        validateEvents(DEVICE_ADDED);
-
-        assertEquals("only one device expected", 1, Iterables.size(service.getDevices()));
-        Iterator<Device> it = service.getDevices().iterator();
-        assertNotNull("one device expected", it.next());
-        assertFalse("only one device expected", it.hasNext());
-
-        assertEquals("incorrect device count", 1, service.getDeviceCount());
-        assertTrue("device should be available", service.isAvailable(DID1));
-    }
-
-    @Test
-    public void deviceDisconnected() {
-        connectDevice(DID1, SW1);
-        connectDevice(DID2, SW1);
-        validateEvents(DEVICE_ADDED, DEVICE_ADDED);
-        assertTrue("device should be available", service.isAvailable(DID1));
-
-        // Disconnect
-        providerService.deviceDisconnected(DID1);
-        assertNotNull("device should not be found", service.getDevice(DID1));
-        assertFalse("device should not be available", service.isAvailable(DID1));
-        validateEvents(DEVICE_AVAILABILITY_CHANGED);
-
-        // Reconnect
-        connectDevice(DID1, SW1);
-        validateEvents(DEVICE_AVAILABILITY_CHANGED);
-
-        assertEquals("incorrect device count", 2, service.getDeviceCount());
-    }
-
-    @Test
-    public void deviceUpdated() {
-        connectDevice(DID1, SW1);
-        validateEvents(DEVICE_ADDED);
-
-        connectDevice(DID1, SW2);
-        validateEvents(DEVICE_UPDATED);
-    }
-
-    @Test
-    public void getRole() {
-        connectDevice(DID1, SW1);
-        assertEquals("incorrect role", MastershipRole.MASTER, service.getRole(DID1));
-    }
-
-    @Test
-    public void updatePorts() {
-        connectDevice(DID1, SW1);
-        List<PortDescription> pds = new ArrayList<>();
-        pds.add(new DefaultPortDescription(P1, true));
-        pds.add(new DefaultPortDescription(P2, true));
-        pds.add(new DefaultPortDescription(P3, true));
-        providerService.updatePorts(DID1, pds);
-        validateEvents(DEVICE_ADDED, PORT_ADDED, PORT_ADDED, PORT_ADDED);
-        pds.clear();
-
-        pds.add(new DefaultPortDescription(P1, false));
-        pds.add(new DefaultPortDescription(P3, true));
-        providerService.updatePorts(DID1, pds);
-        validateEvents(PORT_UPDATED, PORT_REMOVED);
-    }
-
-    @Test
-    public void updatePortStatus() {
-        connectDevice(DID1, SW1);
-        List<PortDescription> pds = new ArrayList<>();
-        pds.add(new DefaultPortDescription(P1, true));
-        pds.add(new DefaultPortDescription(P2, true));
-        providerService.updatePorts(DID1, pds);
-        validateEvents(DEVICE_ADDED, PORT_ADDED, PORT_ADDED);
-
-        providerService.portStatusChanged(DID1, new DefaultPortDescription(P1, false));
-        validateEvents(PORT_UPDATED);
-        providerService.portStatusChanged(DID1, new DefaultPortDescription(P1, false));
-        assertTrue("no events expected", listener.events.isEmpty());
-    }
-
-    @Test
-    public void getPorts() {
-        connectDevice(DID1, SW1);
-        List<PortDescription> pds = new ArrayList<>();
-        pds.add(new DefaultPortDescription(P1, true));
-        pds.add(new DefaultPortDescription(P2, true));
-        providerService.updatePorts(DID1, pds);
-        validateEvents(DEVICE_ADDED, PORT_ADDED, PORT_ADDED);
-        assertEquals("wrong port count", 2, service.getPorts(DID1).size());
-
-        Port port = service.getPort(DID1, P1);
-        assertEquals("incorrect port", P1, service.getPort(DID1, P1).number());
-        assertEquals("incorrect state", true, service.getPort(DID1, P1).isEnabled());
-    }
-
-    @Test
-    public void removeDevice() {
-        connectDevice(DID1, SW1);
-        connectDevice(DID2, SW2);
-        assertEquals("incorrect device count", 2, service.getDeviceCount());
-        admin.removeDevice(DID1);
-        validateEvents(DEVICE_ADDED, DEVICE_ADDED, DEVICE_REMOVED);
-        assertNull("device should not be found", service.getDevice(DID1));
-        assertNotNull("device should be found", service.getDevice(DID2));
-        assertEquals("incorrect device count", 1, service.getDeviceCount());
-    }
-
-    protected void validateEvents(Enum... types) {
-        for (Enum type : types) {
-            try {
-                Event event = listener.events.poll(1, TimeUnit.SECONDS);
-                assertNotNull("Timed out waiting for " + event, event);
-                assertEquals("incorrect event type", type, event.type());
-            } catch (InterruptedException e) {
-                fail("Unexpected interrupt");
-            }
-        }
-        assertTrue("Unexpected events left", listener.events.isEmpty());
-        listener.events.clear();
-    }
-
-
-    private class TestProvider extends AbstractProvider implements DeviceProvider {
-        private Device deviceReceived;
-        private MastershipRole roleReceived;
-
-        public TestProvider() {
-            super(PID);
-        }
-
-        @Override
-        public void triggerProbe(Device device) {
-        }
-
-        @Override
-        public void roleChanged(Device device, MastershipRole newRole) {
-            deviceReceived = device;
-            roleReceived = newRole;
-        }
-    }
-
-    private static class TestListener implements DeviceListener {
-        final BlockingQueue<DeviceEvent> events = new LinkedBlockingQueue<>();
-
-        @Override
-        public void event(DeviceEvent event) {
-            events.add(event);
-        }
-    }
-
-    private class TestDistributedDeviceStore extends DistributedDeviceStore {
-
-        public TestDistributedDeviceStore(StoreService storeService,
-                                    KryoSerializationService kryoSerializationService) {
-            this.storeService = storeService;
-            this.kryoSerializationService = kryoSerializationService;
-        }
-    }
-
-    private static class TestMastershipManager extends MastershipServiceAdapter {
-
-        private ConcurrentMap<DeviceId, NodeId> masters = new ConcurrentHashMap<>();
-
-        public TestMastershipManager() {
-            // SELF master of all initially
-            masters.put(DID1, SELF.id());
-            masters.put(DID1, SELF.id());
-        }
-        @Override
-        public MastershipRole getLocalRole(DeviceId deviceId) {
-            return MastershipRole.MASTER;
-        }
-
-        @Override
-        public Set<DeviceId> getDevicesOf(NodeId nodeId) {
-            HashSet<DeviceId> set = Sets.newHashSet();
-            for (Entry<DeviceId, NodeId> e : masters.entrySet()) {
-                if (e.getValue().equals(nodeId)) {
-                    set.add(e.getKey());
-                }
-            }
-            return set;
-        }
-
-        @Override
-        public MastershipRole requestRoleFor(DeviceId deviceId) {
-            if (SELF.id().equals(masters.get(deviceId))) {
-                return MastershipRole.MASTER;
-            } else {
-                return MastershipRole.STANDBY;
-            }
-        }
-
-        @Override
-        public void relinquishMastership(DeviceId deviceId) {
-            masters.remove(deviceId, SELF.id());
-        }
-    }
-}

@@ -12,6 +12,7 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Set;
+import java.util.concurrent.Future;
 
 import org.junit.After;
 import org.junit.Before;
@@ -27,9 +28,12 @@ import org.onlab.onos.net.Port;
 import org.onlab.onos.net.PortNumber;
 import org.onlab.onos.net.device.DeviceListener;
 import org.onlab.onos.net.device.DeviceService;
+import org.onlab.onos.net.flow.DefaultFlowEntry;
 import org.onlab.onos.net.flow.DefaultFlowRule;
+import org.onlab.onos.net.flow.FlowEntry;
+import org.onlab.onos.net.flow.FlowEntry.FlowEntryState;
 import org.onlab.onos.net.flow.FlowRule;
-import org.onlab.onos.net.flow.FlowRule.FlowRuleState;
+import org.onlab.onos.net.flow.FlowRuleBatchEntry;
 import org.onlab.onos.net.flow.FlowRuleEvent;
 import org.onlab.onos.net.flow.FlowRuleListener;
 import org.onlab.onos.net.flow.FlowRuleProvider;
@@ -40,6 +44,7 @@ import org.onlab.onos.net.flow.TrafficSelector;
 import org.onlab.onos.net.flow.TrafficTreatment;
 import org.onlab.onos.net.flow.criteria.Criterion;
 import org.onlab.onos.net.flow.instructions.Instruction;
+import org.onlab.onos.net.intent.BatchOperation;
 import org.onlab.onos.net.provider.AbstractProvider;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.trivial.impl.SimpleFlowRuleStore;
@@ -100,12 +105,9 @@ public class FlowRuleManagerTest {
     private FlowRule flowRule(int tsval, int trval) {
         TestSelector ts = new TestSelector(tsval);
         TestTreatment tr = new TestTreatment(trval);
-        return new DefaultFlowRule(DID, ts, tr, 0, appId, TIMEOUT);
+        return new DefaultFlowRule(DID, ts, tr, 10, appId, TIMEOUT);
     }
 
-    private FlowRule flowRule(FlowRule rule, FlowRuleState state) {
-        return new DefaultFlowRule(rule, state);
-    }
 
     private FlowRule addFlowRule(int hval) {
         FlowRule rule = flowRule(hval, hval);
@@ -143,24 +145,26 @@ public class FlowRuleManagerTest {
         FlowRule f1 = addFlowRule(1);
         FlowRule f2 = addFlowRule(2);
 
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
         assertEquals("2 rules should exist", 2, flowCount());
 
-        providerService.pushFlowMetrics(DID, ImmutableList.of(f1, f2));
+        providerService.pushFlowMetrics(DID, ImmutableList.of(fe1, fe2));
         validateEvents(RULE_ADDED, RULE_ADDED);
 
         addFlowRule(1);
         assertEquals("should still be 2 rules", 2, flowCount());
 
-        providerService.pushFlowMetrics(DID, ImmutableList.of(f1));
+        providerService.pushFlowMetrics(DID, ImmutableList.of(fe1));
         validateEvents(RULE_UPDATED);
     }
 
 
     //backing store is sensitive to the order of additions/removals
-    private boolean validateState(FlowRuleState... state) {
-        Iterable<FlowRule> rules = service.getFlowEntries(DID);
+    private boolean validateState(FlowEntryState... state) {
+        Iterable<FlowEntry> rules = service.getFlowEntries(DID);
         int i = 0;
-        for (FlowRule f : rules) {
+        for (FlowEntry f : rules) {
             if (f.state() != state[i]) {
                 return false;
             }
@@ -181,8 +185,8 @@ public class FlowRuleManagerTest {
         mgr.applyFlowRules(r1, r2, r3);
         assertEquals("3 rules should exist", 3, flowCount());
         assertTrue("Entries should be pending add.",
-                validateState(FlowRuleState.PENDING_ADD, FlowRuleState.PENDING_ADD,
-                        FlowRuleState.PENDING_ADD));
+                validateState(FlowEntryState.PENDING_ADD, FlowEntryState.PENDING_ADD,
+                        FlowEntryState.PENDING_ADD));
     }
 
     @Test
@@ -192,20 +196,21 @@ public class FlowRuleManagerTest {
         FlowRule f3 = addFlowRule(3);
         assertEquals("3 rules should exist", 3, flowCount());
 
-        providerService.pushFlowMetrics(DID, ImmutableList.of(f1, f2, f3));
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
+        FlowEntry fe3 = new DefaultFlowEntry(f3);
+        providerService.pushFlowMetrics(DID, ImmutableList.of(fe1, fe2, fe3));
         validateEvents(RULE_ADDED, RULE_ADDED, RULE_ADDED);
 
-        FlowRule rem1 = flowRule(f1, FlowRuleState.REMOVED);
-        FlowRule rem2 = flowRule(f2, FlowRuleState.REMOVED);
-        mgr.removeFlowRules(rem1, rem2);
+        mgr.removeFlowRules(f1, f2);
         //removing from north, so no events generated
         validateEvents();
         assertEquals("3 rule should exist", 3, flowCount());
         assertTrue("Entries should be pending remove.",
-                validateState(FlowRuleState.CREATED, FlowRuleState.PENDING_REMOVE,
-                        FlowRuleState.PENDING_REMOVE));
+                validateState(FlowEntryState.PENDING_REMOVE, FlowEntryState.PENDING_REMOVE,
+                        FlowEntryState.ADDED));
 
-        mgr.removeFlowRules(rem1);
+        mgr.removeFlowRules(f1);
         assertEquals("3 rule should still exist", 3, flowCount());
     }
 
@@ -213,21 +218,24 @@ public class FlowRuleManagerTest {
     public void flowRemoved() {
         FlowRule f1 = addFlowRule(1);
         FlowRule f2 = addFlowRule(2);
-        providerService.pushFlowMetrics(f1.deviceId(), ImmutableList.of(f1, f2));
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
+        providerService.pushFlowMetrics(DID, ImmutableList.of(fe1, fe2));
         service.removeFlowRules(f1);
-        FlowRule rem1 = flowRule(f1, FlowRuleState.REMOVED);
-        providerService.flowRemoved(rem1);
+        fe1.setState(FlowEntryState.REMOVED);
+        providerService.flowRemoved(fe1);
         validateEvents(RULE_ADDED, RULE_ADDED, RULE_REMOVED);
 
-        providerService.flowRemoved(rem1);
+        providerService.flowRemoved(fe1);
         validateEvents();
 
         FlowRule f3 = flowRule(3, 3);
+        FlowEntry fe3 = new DefaultFlowEntry(f3);
         service.applyFlowRules(f3);
-        providerService.pushFlowMetrics(f3.deviceId(), Collections.singletonList(f3));
+        providerService.pushFlowMetrics(DID, Collections.singletonList(fe3));
         validateEvents(RULE_ADDED);
 
-        providerService.flowRemoved(f3);
+        providerService.flowRemoved(fe3);
         validateEvents();
     }
 
@@ -237,17 +245,20 @@ public class FlowRuleManagerTest {
         FlowRule f2 = flowRule(2, 2);
         FlowRule f3 = flowRule(3, 3);
 
+        mgr.applyFlowRules(f1, f2, f3);
 
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
 
-        mgr.applyFlowRules(f1, f2, f3);
-        FlowRule updatedF1 = flowRule(f1, FlowRuleState.ADDED);
-        FlowRule updatedF2 = flowRule(f2, FlowRuleState.ADDED);
 
-        providerService.pushFlowMetrics(DID, Lists.newArrayList(updatedF1, updatedF2));
+        //FlowRule updatedF1 = flowRule(f1, FlowRuleState.ADDED);
+        //FlowRule updatedF2 = flowRule(f2, FlowRuleState.ADDED);
+
+        providerService.pushFlowMetrics(DID, Lists.newArrayList(fe1, fe2));
 
         assertTrue("Entries should be added.",
-                validateState(FlowRuleState.PENDING_ADD, FlowRuleState.ADDED,
-                        FlowRuleState.ADDED));
+                validateState(FlowEntryState.ADDED, FlowEntryState.ADDED,
+                        FlowEntryState.PENDING_ADD));
 
         validateEvents(RULE_ADDED, RULE_ADDED);
     }
@@ -259,11 +270,15 @@ public class FlowRuleManagerTest {
         FlowRule f3 = flowRule(3, 3);
         mgr.applyFlowRules(f1, f2);
 
-        FlowRule updatedF1 = flowRule(f1, FlowRuleState.ADDED);
-        FlowRule updatedF2 = flowRule(f2, FlowRuleState.ADDED);
-        FlowRule updatedF3 = flowRule(f3, FlowRuleState.ADDED);
+//        FlowRule updatedF1 = flowRule(f1, FlowRuleState.ADDED);
+//        FlowRule updatedF2 = flowRule(f2, FlowRuleState.ADDED);
+//        FlowRule updatedF3 = flowRule(f3, FlowRuleState.ADDED);
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
+        FlowEntry fe3 = new DefaultFlowEntry(f3);
+
 
-        providerService.pushFlowMetrics(DID, Lists.newArrayList(updatedF1, updatedF2, updatedF3));
+        providerService.pushFlowMetrics(DID, Lists.newArrayList(fe1, fe2, fe3));
 
         validateEvents(RULE_ADDED, RULE_ADDED);
 
@@ -279,13 +294,16 @@ public class FlowRuleManagerTest {
         FlowRule f2 = flowRule(2, 2);
         FlowRule f3 = flowRule(3, 3);
 
-        FlowRule updatedF1 = flowRule(f1, FlowRuleState.ADDED);
-        FlowRule updatedF2 = flowRule(f2, FlowRuleState.ADDED);
+//        FlowRule updatedF1 = flowRule(f1, FlowRuleState.ADDED);
+//        FlowRule updatedF2 = flowRule(f2, FlowRuleState.ADDED);
+
+        FlowEntry fe1 = new DefaultFlowEntry(f1);
+        FlowEntry fe2 = new DefaultFlowEntry(f2);
         mgr.applyFlowRules(f1, f2, f3);
 
         mgr.removeFlowRules(f3);
 
-        providerService.pushFlowMetrics(DID, Lists.newArrayList(updatedF1, updatedF2));
+        providerService.pushFlowMetrics(DID, Lists.newArrayList(fe1, fe2));
 
         validateEvents(RULE_ADDED, RULE_ADDED, RULE_REMOVED);
 
@@ -312,7 +330,7 @@ public class FlowRuleManagerTest {
 
         //only check that we are in pending remove. Events and actual remove state will
         // be set by flowRemoved call.
-        validateState(FlowRuleState.PENDING_REMOVE, FlowRuleState.PENDING_REMOVE);
+        validateState(FlowEntryState.PENDING_REMOVE, FlowEntryState.PENDING_REMOVE);
     }
 
     private static class TestListener implements FlowRuleListener {
@@ -389,6 +407,13 @@ public class FlowRuleManagerTest {
         public void removeRulesById(ApplicationId id, FlowRule... flowRules) {
         }
 
+        @Override
+        public Future<Void> executeBatch(
+                BatchOperation<FlowRuleBatchEntry> batch) {
+            // TODO Auto-generated method stub
+            return null;
+        }
+
 
     }
 

+package org.onlab.onos.net.host.impl;
+
+import static org.easymock.EasyMock.createMock;
+import static org.easymock.EasyMock.expect;
+import static org.easymock.EasyMock.expectLastCall;
+import static org.easymock.EasyMock.replay;
+import static org.easymock.EasyMock.verify;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import org.junit.Ignore;
+import org.junit.Test;
+import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.Device;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.Host;
+import org.onlab.onos.net.MastershipRole;
+import org.onlab.onos.net.Port;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.device.DeviceListener;
+import org.onlab.onos.net.device.DeviceService;
+import org.onlab.onos.net.flow.instructions.Instruction;
+import org.onlab.onos.net.flow.instructions.Instructions.OutputInstruction;
+import org.onlab.onos.net.host.HostProvider;
+import org.onlab.onos.net.host.PortAddresses;
+import org.onlab.onos.net.packet.OutboundPacket;
+import org.onlab.onos.net.packet.PacketProcessor;
+import org.onlab.onos.net.packet.PacketService;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.packet.ARP;
+import org.onlab.packet.Ethernet;
+import org.onlab.packet.IpAddress;
+import org.onlab.packet.IpPrefix;
+import org.onlab.packet.MacAddress;
+
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+
+public class HostMonitorTest {
+
+    private IpAddress targetIpAddress = IpAddress.valueOf("10.0.0.1");
+    private IpPrefix targetIpPrefix = IpPrefix.valueOf(targetIpAddress.toOctets());
+
+    private IpPrefix sourcePrefix = IpPrefix.valueOf("10.0.0.99/24");
+    private MacAddress sourceMac = MacAddress.valueOf(1L);
+
+    private HostMonitor hostMonitor;
+
+    @Test
+    @Ignore
+    public void testMonitorHostExists() throws Exception {
+        ProviderId id = new ProviderId("fake://", "id");
+
+        Host host = createMock(Host.class);
+        expect(host.providerId()).andReturn(id);
+        replay(host);
+
+        HostManager hostManager = createMock(HostManager.class);
+        expect(hostManager.getHostsByIp(targetIpPrefix))
+                .andReturn(Collections.singleton(host));
+        replay(hostManager);
+
+        HostProvider hostProvider = createMock(HostProvider.class);
+        expect(hostProvider.id()).andReturn(id).anyTimes();
+        hostProvider.triggerProbe(host);
+        expectLastCall().once();
+        replay(hostProvider);
+
+        hostMonitor = new HostMonitor(null, null, hostManager);
+
+        hostMonitor.registerHostProvider(hostProvider);
+        hostMonitor.addMonitoringFor(targetIpAddress);
+
+        hostMonitor.run(null);
+
+        verify(hostProvider);
+    }
+
+    @Test
+    @Ignore
+    public void testMonitorHostDoesNotExist() throws Exception {
+        HostManager hostManager = createMock(HostManager.class);
+
+        DeviceId devId = DeviceId.deviceId("fake");
+
+        Device device = createMock(Device.class);
+        expect(device.id()).andReturn(devId).anyTimes();
+        replay(device);
+
+        PortNumber portNum = PortNumber.portNumber(1L);
+
+        Port port = createMock(Port.class);
+        expect(port.number()).andReturn(portNum).anyTimes();
+        replay(port);
+
+        TestDeviceService deviceService = new TestDeviceService();
+        deviceService.addDevice(device, Collections.singleton(port));
+
+        ConnectPoint cp = new ConnectPoint(devId, portNum);
+        PortAddresses pa = new PortAddresses(cp, Collections.singleton(sourcePrefix),
+                sourceMac);
+
+        expect(hostManager.getHostsByIp(targetIpPrefix))
+                .andReturn(Collections.<Host>emptySet()).anyTimes();
+        expect(hostManager.getAddressBindingsForPort(cp))
+                .andReturn(pa).anyTimes();
+        replay(hostManager);
+
+        TestPacketService packetService = new TestPacketService();
+
+
+        // Run the test
+        hostMonitor = new HostMonitor(deviceService, packetService, hostManager);
+
+        hostMonitor.addMonitoringFor(targetIpAddress);
+        hostMonitor.run(null);
+
+
+        // Check that a packet was sent to our PacketService and that it has
+        // the properties we expect
+        assertTrue(packetService.packets.size() == 1);
+        OutboundPacket packet = packetService.packets.get(0);
+
+        // Check the output port is correct
+        assertTrue(packet.treatment().instructions().size() == 1);
+        Instruction instruction = packet.treatment().instructions().get(0);
+        assertTrue(instruction instanceof OutputInstruction);
+        OutputInstruction oi = (OutputInstruction) instruction;
+        assertTrue(oi.port().equals(portNum));
+
+        // Check the output packet is correct (well the important bits anyway)
+        Ethernet eth = new Ethernet();
+        eth.deserialize(packet.data().array(), 0, packet.data().array().length);
+        ARP arp = (ARP) eth.getPayload();
+        assertTrue(Arrays.equals(arp.getSenderProtocolAddress(), sourcePrefix.toOctets()));
+        assertTrue(Arrays.equals(arp.getSenderHardwareAddress(), sourceMac.toBytes()));
+        assertTrue(Arrays.equals(arp.getTargetProtocolAddress(), targetIpPrefix.toOctets()));
+    }
+
+    class TestPacketService implements PacketService {
+
+        List<OutboundPacket> packets = new ArrayList<>();
+
+        @Override
+        public void addProcessor(PacketProcessor processor, int priority) {
+        }
+
+        @Override
+        public void removeProcessor(PacketProcessor processor) {
+        }
+
+        @Override
+        public void emit(OutboundPacket packet) {
+            packets.add(packet);
+        }
+    }
+
+    class TestDeviceService implements DeviceService {
+
+        List<Device> devices = Lists.newArrayList();
+        Multimap<DeviceId, Port> devicePorts = HashMultimap.create();
+
+        void addDevice(Device device, Set<Port> ports) {
+            devices.add(device);
+            for (Port p : ports) {
+                devicePorts.put(device.id(), p);
+            }
+        }
+
+        @Override
+        public int getDeviceCount() {
+            return 0;
+        }
+
+        @Override
+        public Iterable<Device> getDevices() {
+            return devices;
+        }
+
+        @Override
+        public Device getDevice(DeviceId deviceId) {
+            return null;
+        }
+
+        @Override
+        public MastershipRole getRole(DeviceId deviceId) {
+            return null;
+        }
+
+        @Override
+        public List<Port> getPorts(DeviceId deviceId) {
+            List<Port> ports = Lists.newArrayList();
+            for (Port p : devicePorts.get(deviceId)) {
+                ports.add(p);
+            }
+            return ports;
+        }
+
+        @Override
+        public Port getPort(DeviceId deviceId, PortNumber portNumber) {
+            return null;
+        }
+
+        @Override
+        public boolean isAvailable(DeviceId deviceId) {
+            return false;
+        }
+
+        @Override
+        public void addListener(DeviceListener listener) {
+        }
+
+        @Override
+        public void removeListener(DeviceListener listener) {
+        }
+    }
+}

-package org.onlab.onos.store.cluster.messaging.impl;
+package org.onlab.onos.store.cluster.impl;
 
 import org.onlab.onos.store.cluster.messaging.MessageSubject;
 
-public final class ClusterMessageSubjects {
-    private ClusterMessageSubjects() {}
+public final class ClusterManagementMessageSubjects {
+    // avoid instantiation
+    private ClusterManagementMessageSubjects() {}
+
     public static final MessageSubject CLUSTER_MEMBERSHIP_EVENT = new MessageSubject("CLUSTER_MEMBERSHIP_EVENT");
 }

-package org.onlab.onos.store.cluster.messaging.impl;
+package org.onlab.onos.store.cluster.impl;
 
 import org.onlab.onos.cluster.ControllerNode;
 

-package org.onlab.onos.store.cluster.messaging.impl;
+package org.onlab.onos.store.cluster.impl;
 
 public enum ClusterMembershipEventType {
     NEW_MEMBER,

@@ -7,11 +7,9 @@ import com.google.common.cache.RemovalNotification;
 import com.google.common.collect.ImmutableSet;
 
 import org.apache.felix.scr.annotations.Activate;
-import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
-import org.apache.felix.scr.annotations.Service;
 import org.onlab.onos.cluster.ClusterEvent;
 import org.onlab.onos.cluster.ClusterStore;
 import org.onlab.onos.cluster.ClusterStoreDelegate;
@@ -37,8 +35,8 @@ import static org.onlab.packet.IpPrefix.valueOf;
 /**
  * Distributed implementation of the cluster nodes store.
  */
-@Component(immediate = true)
-@Service
+//@Component(immediate = true)
+//@Service
 public class DistributedClusterStore
         extends AbstractStore<ClusterEvent, ClusterStoreDelegate>
         implements ClusterStore {

@@ -3,6 +3,8 @@ package org.onlab.onos.store.cluster.messaging;
 import org.onlab.onos.cluster.ControllerNode;
 import org.onlab.onos.store.cluster.impl.ClusterNodesDelegate;
 
+// TODO: This service interface can be removed, once we properly start
+// using ClusterService
 /**
  * Service for administering communications manager.
  */

@@ -2,6 +2,7 @@ package org.onlab.onos.store.cluster.messaging;
 
 import org.onlab.onos.cluster.NodeId;
 
+// TODO: Should payload type be ByteBuffer?
 /**
  * Base message for cluster-wide communications.
  */
@@ -9,14 +10,14 @@ public class ClusterMessage {
 
     private final NodeId sender;
     private final MessageSubject subject;
-    private final Object payload;
+    private final byte[] payload;
 
     /**
      * Creates a cluster message.
      *
      * @param subject message subject
      */
-    public ClusterMessage(NodeId sender, MessageSubject subject, Object payload) {
+    public ClusterMessage(NodeId sender, MessageSubject subject, byte[] payload) {
         this.sender = sender;
         this.subject = subject;
         this.payload = payload;
@@ -45,7 +46,7 @@ public class ClusterMessage {
      *
      * @return message payload.
      */
-    public Object payload() {
+    public byte[] payload() {
         return payload;
     }
 }

@@ -10,4 +10,4 @@ public interface ClusterMessageHandler {
      * @param message cluster message.
      */
     public void handle(ClusterMessage message);
-}
\ No newline at end of file
+}

 package org.onlab.onos.store.cluster.messaging;
 
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Objects;
+
 /**
  * Representation of a message subject.
  * Cluster messages have associated subjects that dictate how they get handled
@@ -10,7 +14,7 @@ public class MessageSubject {
     private final String value;
 
     public MessageSubject(String value) {
-        this.value = value;
+        this.value = checkNotNull(value);
     }
 
     public String value() {
@@ -21,4 +25,29 @@ public class MessageSubject {
     public String toString() {
         return value;
     }
+
+    @Override
+    public int hashCode() {
+        return value.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null) {
+            return false;
+        }
+        if (getClass() != obj.getClass()) {
+            return false;
+        }
+        MessageSubject that = (MessageSubject) obj;
+        return Objects.equals(this.value, that.value);
+    }
+
+    // for serializer
+    protected MessageSubject() {
+        this.value = "";
+    }
 }

 package org.onlab.onos.store.cluster.messaging;
 
 /**
- * Service for encoding & decoding intra-cluster messages.
+ * Service for encoding & decoding intra-cluster message payload.
  */
 public interface SerializationService {
 
@@ -11,7 +11,7 @@ public interface SerializationService {
      * @param buffer byte buffer with message(s)
      * @return parsed message
      */
-    Object decode(byte[] data);
+    <T> T decode(byte[] data);
 
     /**
      * Encodes the specified message into the given byte buffer.

@@ -3,30 +3,36 @@ package org.onlab.onos.store.cluster.messaging.impl;
 import static com.google.common.base.Preconditions.checkArgument;
 
 import java.io.IOException;
-import java.util.HashMap;
-import java.util.Map;
 import java.util.Set;
 import java.util.Timer;
 import java.util.TimerTask;
-
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.cluster.ClusterService;
 import org.onlab.onos.cluster.ControllerNode;
 import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.store.cluster.impl.ClusterMembershipEvent;
+import org.onlab.onos.store.cluster.impl.ClusterMembershipEventType;
 import org.onlab.onos.store.cluster.impl.ClusterNodesDelegate;
 import org.onlab.onos.store.cluster.messaging.ClusterCommunicationAdminService;
 import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
 import org.onlab.onos.store.cluster.messaging.ClusterMessage;
 import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
 import org.onlab.onos.store.cluster.messaging.MessageSubject;
+import org.onlab.onos.store.serializers.ClusterMessageSerializer;
+import org.onlab.onos.store.serializers.KryoPoolUtil;
+import org.onlab.onos.store.serializers.KryoSerializer;
+import org.onlab.onos.store.serializers.MessageSubjectSerializer;
+import org.onlab.util.KryoPool;
 import org.onlab.netty.Endpoint;
 import org.onlab.netty.Message;
 import org.onlab.netty.MessageHandler;
 import org.onlab.netty.MessagingService;
+import org.onlab.netty.NettyMessagingService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,28 +44,57 @@ public class ClusterCommunicationManager
     private final Logger log = LoggerFactory.getLogger(getClass());
 
     private ControllerNode localNode;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    private ClusterService clusterService;
+
     private ClusterNodesDelegate nodesDelegate;
-    private Map<NodeId, ControllerNode> members = new HashMap<>();
     private final Timer timer = new Timer("onos-controller-heatbeats");
     public static final long HEART_BEAT_INTERVAL_MILLIS = 1000L;
 
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    // TODO: This probably should not be a OSGi service.
     private MessagingService messagingService;
 
+    private static final KryoSerializer SERIALIZER = new KryoSerializer() {
+        @Override
+        protected void setupKryoPool() {
+            serializerPool = KryoPool.newBuilder()
+                    .register(KryoPoolUtil.API)
+                    .register(ClusterMessage.class, new ClusterMessageSerializer())
+                    .register(ClusterMembershipEvent.class)
+                    .register(byte[].class)
+                    .register(MessageSubject.class, new MessageSubjectSerializer())
+                    .build()
+                    .populate(1);
+        }
+
+    };
+
     @Activate
     public void activate() {
+        localNode = clusterService.getLocalNode();
+        NettyMessagingService netty = new NettyMessagingService(localNode.tcpPort());
+        // FIXME: workaround until it becomes a service.
+        try {
+            netty.activate();
+        } catch (Exception e) {
+            // TODO Auto-generated catch block
+            log.error("NettyMessagingService#activate", e);
+        }
+        messagingService = netty;
         log.info("Started");
     }
 
     @Deactivate
     public void deactivate() {
+        // TODO: cleanup messageingService if needed.
         log.info("Stopped");
     }
 
     @Override
     public boolean broadcast(ClusterMessage message) {
         boolean ok = true;
-        for (ControllerNode node : members.values()) {
+        for (ControllerNode node : clusterService.getNodes()) {
             if (!node.equals(localNode)) {
                 ok = unicast(message, node.id()) && ok;
             }
@@ -80,11 +115,12 @@ public class ClusterCommunicationManager
 
     @Override
     public boolean unicast(ClusterMessage message, NodeId toNodeId) {
-        ControllerNode node = members.get(toNodeId);
+        ControllerNode node = clusterService.getNode(toNodeId);
         checkArgument(node != null, "Unknown nodeId: %s", toNodeId);
         Endpoint nodeEp = new Endpoint(node.ip().toString(), node.tcpPort());
         try {
-            messagingService.sendAsync(nodeEp, message.subject().value(), message);
+            messagingService.sendAsync(nodeEp,
+                    message.subject().value(), SERIALIZER.encode(message));
             return true;
         } catch (IOException e) {
             log.error("Failed to send cluster message to nodeId: " + toNodeId, e);
@@ -110,7 +146,7 @@ public class ClusterCommunicationManager
 
     @Override
     public void addNode(ControllerNode node) {
-        members.put(node.id(), node);
+        //members.put(node.id(), node);
     }
 
     @Override
@@ -118,8 +154,8 @@ public class ClusterCommunicationManager
         broadcast(new ClusterMessage(
                 localNode.id(),
                 new MessageSubject("CLUSTER_MEMBERSHIP_EVENT"),
-                new ClusterMembershipEvent(ClusterMembershipEventType.LEAVING_MEMBER, node)));
-        members.remove(node.id());
+                SERIALIZER.encode(new ClusterMembershipEvent(ClusterMembershipEventType.LEAVING_MEMBER, node))));
+        //members.remove(node.id());
     }
 
     // Sends a heart beat to all peers.
@@ -130,7 +166,7 @@ public class ClusterCommunicationManager
             broadcast(new ClusterMessage(
                 localNode.id(),
                 new MessageSubject("CLUSTER_MEMBERSHIP_EVENT"),
-                new ClusterMembershipEvent(ClusterMembershipEventType.HEART_BEAT, localNode)));
+                SERIALIZER.encode(new ClusterMembershipEvent(ClusterMembershipEventType.HEART_BEAT, localNode))));
         }
     }
 
@@ -139,7 +175,7 @@ public class ClusterCommunicationManager
         @Override
         public void handle(ClusterMessage message) {
 
-            ClusterMembershipEvent event = (ClusterMembershipEvent) message.payload();
+            ClusterMembershipEvent event = SERIALIZER.decode(message.payload());
             ControllerNode node = event.node();
             if (event.type() == ClusterMembershipEventType.HEART_BEAT) {
                 log.info("Node {} sent a hearbeat", node.id());
@@ -154,7 +190,7 @@ public class ClusterCommunicationManager
         }
     }
 
-    private static class InternalClusterMessageHandler implements MessageHandler {
+    private final class InternalClusterMessageHandler implements MessageHandler {
 
         private final ClusterMessageHandler handler;
 
@@ -164,7 +200,13 @@ public class ClusterCommunicationManager
 
         @Override
         public void handle(Message message) {
-            handler.handle((ClusterMessage) message.payload());
+            try {
+                ClusterMessage clusterMessage = SERIALIZER.decode(message.payload());
+                handler.handle(clusterMessage);
+            } catch (Exception e) {
+                log.error("Exception caught during ClusterMessageHandler", e);
+                throw e;
+            }
         }
     }
 }

-package org.onlab.onos.store.cluster.impl;
+package org.onlab.onos.store.cluster.messaging.impl;
 
-import de.javakaffee.kryoserializers.URISerializer;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
-import org.onlab.onos.cluster.ControllerNode;
-import org.onlab.onos.cluster.DefaultControllerNode;
-import org.onlab.onos.cluster.NodeId;
-import org.onlab.onos.net.ConnectPoint;
-import org.onlab.onos.net.DefaultDevice;
-import org.onlab.onos.net.DefaultLink;
-import org.onlab.onos.net.DefaultPort;
-import org.onlab.onos.net.Device;
-import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.net.Element;
-import org.onlab.onos.net.Link;
-import org.onlab.onos.net.LinkKey;
-import org.onlab.onos.net.MastershipRole;
-import org.onlab.onos.net.Port;
-import org.onlab.onos.net.PortNumber;
-import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.cluster.messaging.MessageSubject;
 import org.onlab.onos.store.cluster.messaging.SerializationService;
-import org.onlab.onos.store.serializers.ConnectPointSerializer;
-import org.onlab.onos.store.serializers.DefaultLinkSerializer;
-import org.onlab.onos.store.serializers.DefaultPortSerializer;
-import org.onlab.onos.store.serializers.DeviceIdSerializer;
-import org.onlab.onos.store.serializers.IpPrefixSerializer;
-import org.onlab.onos.store.serializers.LinkKeySerializer;
-import org.onlab.onos.store.serializers.NodeIdSerializer;
-import org.onlab.onos.store.serializers.PortNumberSerializer;
-import org.onlab.onos.store.serializers.ProviderIdSerializer;
-import org.onlab.packet.IpPrefix;
+import org.onlab.onos.store.serializers.KryoPoolUtil;
 import org.onlab.util.KryoPool;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashMap;
-
 /**
  * Factory for parsing messages sent between cluster members.
  */
@@ -72,41 +42,17 @@ public class MessageSerializer implements SerializationService {
      * Sets up the common serialzers pool.
      */
     protected void setupKryoPool() {
-        // FIXME Slice out types used in common to separate pool/namespace.
         serializerPool = KryoPool.newBuilder()
-                .register(ArrayList.class,
-                          HashMap.class,
-
-                          ControllerNode.State.class,
-                          Device.Type.class,
-
-                          DefaultControllerNode.class,
-                          DefaultDevice.class,
-                          MastershipRole.class,
-                          Port.class,
-                          Element.class,
-
-                          Link.Type.class,
-
-                          MessageSubject.class
-                )
-                .register(IpPrefix.class, new IpPrefixSerializer())
-                .register(URI.class, new URISerializer())
-                .register(NodeId.class, new NodeIdSerializer())
-                .register(ProviderId.class, new ProviderIdSerializer())
-                .register(DeviceId.class, new DeviceIdSerializer())
-                .register(PortNumber.class, new PortNumberSerializer())
-                .register(DefaultPort.class, new DefaultPortSerializer())
-                .register(LinkKey.class, new LinkKeySerializer())
-                .register(ConnectPoint.class, new ConnectPointSerializer())
-                .register(DefaultLink.class, new DefaultLinkSerializer())
+                .register(KryoPoolUtil.API)
+                // TODO: Should MessageSubject be in API bundle?
+                .register(MessageSubject.class)
                 .build()
                 .populate(1);
     }
 
 
     @Override
-    public Object decode(byte[] data) {
+    public <T> T decode(byte[] data) {
         return serializerPool.deserialize(data);
     }
 
@@ -114,4 +60,4 @@ public class MessageSerializer implements SerializationService {
     public byte[] encode(Object payload) {
         return serializerPool.serialize(payload);
     }
-}
\ No newline at end of file
+}

+/**
+ * Implementation of the cluster messaging mechanism.
+ */
+package org.onlab.onos.store.cluster.messaging.impl;
\ No newline at end of file

-package org.onlab.onos.store.cluster.messaging;
+package org.onlab.onos.store.common.impl;
 
 import java.util.Map;
 

-package org.onlab.onos.store.cluster.messaging;
+package org.onlab.onos.store.common.impl;
 
 import java.util.Map;
 import java.util.Set;

-package org.onlab.onos.store.impl;
+package org.onlab.onos.store.common.impl;
 
 import static com.google.common.base.Preconditions.checkArgument;
 
@@ -9,12 +9,11 @@ import org.onlab.onos.store.Timestamp;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.ComparisonChain;
 
-// If it is store specific, implement serializable interfaces?
 /**
  * Default implementation of Timestamp.
  * TODO: Better documentation.
  */
-public final class OnosTimestamp implements Timestamp {
+public final class MastershipBasedTimestamp implements Timestamp {
 
     private final int termNumber;
     private final int sequenceNumber;
@@ -25,15 +24,16 @@ public final class OnosTimestamp implements Timestamp {
      * @param termNumber the mastership termNumber
      * @param sequenceNumber  the sequenceNumber number within the termNumber
      */
-    public OnosTimestamp(int termNumber, int sequenceNumber) {
+    public MastershipBasedTimestamp(int termNumber, int sequenceNumber) {
         this.termNumber = termNumber;
         this.sequenceNumber = sequenceNumber;
     }
 
     @Override
     public int compareTo(Timestamp o) {
-        checkArgument(o instanceof OnosTimestamp, "Must be OnosTimestamp", o);
-        OnosTimestamp that = (OnosTimestamp) o;
+        checkArgument(o instanceof MastershipBasedTimestamp,
+                "Must be MastershipBasedTimestamp", o);
+        MastershipBasedTimestamp that = (MastershipBasedTimestamp) o;
 
         return ComparisonChain.start()
                 .compare(this.termNumber, that.termNumber)
@@ -51,10 +51,10 @@ public final class OnosTimestamp implements Timestamp {
         if (this == obj) {
             return true;
         }
-        if (!(obj instanceof OnosTimestamp)) {
+        if (!(obj instanceof MastershipBasedTimestamp)) {
             return false;
         }
-        OnosTimestamp that = (OnosTimestamp) obj;
+        MastershipBasedTimestamp that = (MastershipBasedTimestamp) obj;
         return Objects.equals(this.termNumber, that.termNumber) &&
                 Objects.equals(this.sequenceNumber, that.sequenceNumber);
     }
@@ -84,4 +84,11 @@ public final class OnosTimestamp implements Timestamp {
     public int sequenceNumber() {
         return sequenceNumber;
     }
+
+    // Default constructor for serialization
+    @Deprecated
+    protected MastershipBasedTimestamp() {
+        this.termNumber = -1;
+        this.sequenceNumber = -1;
+    }
 }

+package org.onlab.onos.store.common.impl;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.Objects;
+
+import org.onlab.onos.store.Timestamp;
+
+import com.google.common.base.MoreObjects;
+
+/**
+ * Wrapper class to store Timestamped value.
+ * @param <T>
+ */
+public final class Timestamped<T> {
+
+    private final Timestamp timestamp;
+    private final T value;
+
+    /**
+     * Creates a time stamped value.
+     *
+     * @param value to be timestamp
+     * @param timestamp the timestamp
+     */
+    public Timestamped(T value, Timestamp timestamp) {
+        this.value = checkNotNull(value);
+        this.timestamp = checkNotNull(timestamp);
+    }
+
+    /**
+     * Returns the value.
+     * @return value
+     */
+    public T value() {
+        return value;
+    }
+
+    /**
+     * Returns the time stamp.
+     * @return time stamp
+     */
+    public Timestamp timestamp() {
+        return timestamp;
+    }
+
+    /**
+     * Tests if this timestamped value is newer than the other.
+     *
+     * @param other timestamped value
+     * @return true if this instance is newer.
+     */
+    public boolean isNewer(Timestamped<T> other) {
+        return this.timestamp.compareTo(checkNotNull(other).timestamp()) > 0;
+    }
+
+    @Override
+    public int hashCode() {
+        return timestamp.hashCode();
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof Timestamped)) {
+            return false;
+        }
+        @SuppressWarnings("unchecked")
+        Timestamped<T> that = (Timestamped<T>) obj;
+        return Objects.equals(this.timestamp, that.timestamp);
+    }
+
+    @Override
+    public String toString() {
+        return MoreObjects.toStringHelper(getClass())
+                    .add("timestamp", timestamp)
+                    .add("value", value)
+                    .toString();
+    }
+
+    // Default constructor for serialization
+    @Deprecated
+    private Timestamped() {
+        this.value = null;
+        this.timestamp = null;
+    }
+}

+/**
+ * Common abstractions and facilities for implementing distributed store
+ * using gossip protocol.
+ */
+package org.onlab.onos.store.common.impl;

@@ -8,7 +8,7 @@ import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.store.Timestamp;
-import org.onlab.onos.store.cluster.messaging.AntiEntropyAdvertisement;
+import org.onlab.onos.store.common.impl.AntiEntropyAdvertisement;
 
 // TODO DeviceID needs to be changed to something like (ProviderID, DeviceID)
 // TODO: Handle Port as part of these messages, or separate messages for Ports?

@@ -10,7 +10,7 @@ import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.net.Device;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.store.Timestamp;
-import org.onlab.onos.store.cluster.messaging.AntiEntropyReply;
+import org.onlab.onos.store.common.impl.AntiEntropyReply;
 
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;

@@ -12,14 +12,18 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.onos.cluster.MastershipTerm;
 import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.store.ClockProviderService;
 import org.onlab.onos.store.ClockService;
 import org.onlab.onos.store.Timestamp;
-import org.onlab.onos.store.impl.OnosTimestamp;
+import org.onlab.onos.store.common.impl.MastershipBasedTimestamp;
 import org.slf4j.Logger;
 
+/**
+ * Clock service to issue Timestamp based on Device Mastership.
+ */
 @Component(immediate = true)
 @Service
-public class OnosClockService implements ClockService {
+public class DeviceClockManager implements ClockService, ClockProviderService {
 
     private final Logger log = getLogger(getClass());
 
@@ -43,7 +47,7 @@ public class OnosClockService implements ClockService {
         if (term == null) {
             throw new IllegalStateException("Requesting timestamp for a deviceId without mastership");
         }
-        return new OnosTimestamp(term.termNumber(), ticker.incrementAndGet());
+        return new MastershipBasedTimestamp(term.termNumber(), ticker.incrementAndGet());
     }
 
     @Override

+package org.onlab.onos.store.device.impl;
+
+import com.google.common.collect.FluentIterable;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import org.apache.commons.lang3.concurrent.ConcurrentException;
+import org.apache.commons.lang3.concurrent.ConcurrentInitializer;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Reference;
+import org.apache.felix.scr.annotations.ReferenceCardinality;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.cluster.ClusterService;
+import org.onlab.onos.net.AnnotationsUtil;
+import org.onlab.onos.net.DefaultAnnotations;
+import org.onlab.onos.net.DefaultDevice;
+import org.onlab.onos.net.DefaultPort;
+import org.onlab.onos.net.Device;
+import org.onlab.onos.net.Device.Type;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.Port;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.SparseAnnotations;
+import org.onlab.onos.net.device.DefaultDeviceDescription;
+import org.onlab.onos.net.device.DefaultPortDescription;
+import org.onlab.onos.net.device.DeviceDescription;
+import org.onlab.onos.net.device.DeviceEvent;
+import org.onlab.onos.net.device.DeviceStore;
+import org.onlab.onos.net.device.DeviceStoreDelegate;
+import org.onlab.onos.net.device.PortDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.AbstractStore;
+import org.onlab.onos.store.ClockService;
+import org.onlab.onos.store.Timestamp;
+import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
+import org.onlab.onos.store.cluster.messaging.ClusterMessage;
+import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
+import org.onlab.onos.store.common.impl.MastershipBasedTimestamp;
+import org.onlab.onos.store.common.impl.Timestamped;
+import org.onlab.onos.store.serializers.KryoPoolUtil;
+import org.onlab.onos.store.serializers.KryoSerializer;
+import org.onlab.onos.store.serializers.MastershipBasedTimestampSerializer;
+import org.onlab.util.KryoPool;
+import org.onlab.util.NewConcurrentHashMap;
+import org.slf4j.Logger;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkNotNull;
+import static com.google.common.base.Predicates.notNull;
+import static org.onlab.onos.net.device.DeviceEvent.Type.*;
+import static org.slf4j.LoggerFactory.getLogger;
+import static org.apache.commons.lang3.concurrent.ConcurrentUtils.createIfAbsentUnchecked;
+import static org.onlab.onos.net.DefaultAnnotations.merge;
+import static org.onlab.onos.net.DefaultAnnotations.union;
+import static com.google.common.base.Verify.verify;
+
+// TODO: give me a better name
+/**
+ * Manages inventory of infrastructure devices using gossip protocol to distribute
+ * information.
+ */
+@Component(immediate = true)
+@Service
+public class GossipDeviceStore
+        extends AbstractStore<DeviceEvent, DeviceStoreDelegate>
+        implements DeviceStore {
+
+    private final Logger log = getLogger(getClass());
+
+    public static final String DEVICE_NOT_FOUND = "Device with ID %s not found";
+
+    // TODO: Check if inner Map can be replaced with plain Map
+    // innerMap is used to lock a Device, thus instance should never be replaced.
+    // collection of Description given from various providers
+    private final ConcurrentMap<DeviceId,
+                            ConcurrentMap<ProviderId, DeviceDescriptions>>
+                                deviceDescs = Maps.newConcurrentMap();
+
+    // cache of Device and Ports generated by compositing descriptions from providers
+    private final ConcurrentMap<DeviceId, Device> devices = Maps.newConcurrentMap();
+    private final ConcurrentMap<DeviceId, ConcurrentMap<PortNumber, Port>> devicePorts = Maps.newConcurrentMap();
+
+    // to be updated under Device lock
+    private final Map<DeviceId, Timestamp> offline = Maps.newHashMap();
+    private final Map<DeviceId, Timestamp> removalRequest = Maps.newHashMap();
+
+    // available(=UP) devices
+    private final Set<DeviceId> availableDevices = Sets.newConcurrentHashSet();
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClockService clockService;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterCommunicationService clusterCommunicator;
+
+    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
+    protected ClusterService clusterService;
+
+    private static final KryoSerializer SERIALIZER = new KryoSerializer() {
+        @Override
+        protected void setupKryoPool() {
+            serializerPool = KryoPool.newBuilder()
+                    .register(KryoPoolUtil.API)
+                    .register(InternalDeviceEvent.class, new InternalDeviceEventSerializer())
+                    .register(InternalDeviceOfflineEvent.class, new InternalDeviceOfflineEventSerializer())
+                    .register(InternalDeviceRemovedEvent.class)
+                    .register(InternalPortEvent.class, new InternalPortEventSerializer())
+                    .register(InternalPortStatusEvent.class, new InternalPortStatusEventSerializer())
+                    .register(Timestamp.class)
+                    .register(Timestamped.class)
+                    .register(MastershipBasedTimestamp.class, new MastershipBasedTimestampSerializer())
+                    .build()
+                    .populate(1);
+        }
+
+    };
+
+    @Activate
+    public void activate() {
+        clusterCommunicator.addSubscriber(
+                GossipDeviceStoreMessageSubjects.DEVICE_UPDATE, new InternalDeviceEventListener());
+        clusterCommunicator.addSubscriber(
+                GossipDeviceStoreMessageSubjects.DEVICE_OFFLINE, new InternalDeviceOfflineEventListener());
+        clusterCommunicator.addSubscriber(
+                GossipDeviceStoreMessageSubjects.DEVICE_REMOVED, new InternalDeviceRemovedEventListener());
+        clusterCommunicator.addSubscriber(
+                GossipDeviceStoreMessageSubjects.PORT_UPDATE, new InternalPortEventListener());
+        clusterCommunicator.addSubscriber(
+                GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE, new InternalPortStatusEventListener());
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        deviceDescs.clear();
+        devices.clear();
+        devicePorts.clear();
+        availableDevices.clear();
+        log.info("Stopped");
+    }
+
+    @Override
+    public int getDeviceCount() {
+        return devices.size();
+    }
+
+    @Override
+    public Iterable<Device> getDevices() {
+        return Collections.unmodifiableCollection(devices.values());
+    }
+
+    @Override
+    public Device getDevice(DeviceId deviceId) {
+        return devices.get(deviceId);
+    }
+
+    @Override
+    public synchronized DeviceEvent createOrUpdateDevice(ProviderId providerId,
+                                     DeviceId deviceId,
+                                     DeviceDescription deviceDescription) {
+        Timestamp newTimestamp = clockService.getTimestamp(deviceId);
+        final Timestamped<DeviceDescription> deltaDesc = new Timestamped<>(deviceDescription, newTimestamp);
+        DeviceEvent event = createOrUpdateDeviceInternal(providerId, deviceId, deltaDesc);
+        if (event != null) {
+            log.info("Notifying peers of a device update topology event for providerId: {} and deviceId: {}",
+                providerId, deviceId);
+            try {
+                notifyPeers(new InternalDeviceEvent(providerId, deviceId, deltaDesc));
+            } catch (IOException e) {
+                log.error("Failed to notify peers of a device update topology event for providerId: "
+                        + providerId + " and deviceId: " + deviceId, e);
+            }
+        }
+        return event;
+    }
+
+    private DeviceEvent createOrUpdateDeviceInternal(ProviderId providerId,
+                                    DeviceId deviceId,
+                                    Timestamped<DeviceDescription> deltaDesc) {
+
+        // Collection of DeviceDescriptions for a Device
+        ConcurrentMap<ProviderId, DeviceDescriptions> providerDescs
+            = getDeviceDescriptions(deviceId);
+
+        synchronized (providerDescs) {
+            // locking per device
+
+            if (isDeviceRemoved(deviceId, deltaDesc.timestamp())) {
+                log.debug("Ignoring outdated event: {}", deltaDesc);
+                return null;
+            }
+
+            DeviceDescriptions descs
+                = createIfAbsentUnchecked(providerDescs, providerId,
+                    new InitDeviceDescs(deltaDesc));
+
+            final Device oldDevice = devices.get(deviceId);
+            final Device newDevice;
+
+            if (deltaDesc == descs.getDeviceDesc() ||
+                deltaDesc.isNewer(descs.getDeviceDesc())) {
+                // on new device or valid update
+                descs.putDeviceDesc(deltaDesc);
+                newDevice = composeDevice(deviceId, providerDescs);
+            } else {
+                // outdated event, ignored.
+                return null;
+            }
+            if (oldDevice == null) {
+                // ADD
+                return createDevice(providerId, newDevice, deltaDesc.timestamp());
+            } else {
+                // UPDATE or ignore (no change or stale)
+                return updateDevice(providerId, oldDevice, newDevice, deltaDesc.timestamp());
+            }
+        }
+    }
+
+    // Creates the device and returns the appropriate event if necessary.
+    // Guarded by deviceDescs value (=Device lock)
+    private DeviceEvent createDevice(ProviderId providerId,
+                                     Device newDevice, Timestamp timestamp) {
+
+        // update composed device cache
+        Device oldDevice = devices.putIfAbsent(newDevice.id(), newDevice);
+        verify(oldDevice == null,
+                "Unexpected Device in cache. PID:%s [old=%s, new=%s]",
+                providerId, oldDevice, newDevice);
+
+        if (!providerId.isAncillary()) {
+            markOnline(newDevice.id(), timestamp);
+        }
+
+        return new DeviceEvent(DeviceEvent.Type.DEVICE_ADDED, newDevice, null);
+    }
+
+    // Updates the device and returns the appropriate event if necessary.
+    // Guarded by deviceDescs value (=Device lock)
+    private DeviceEvent updateDevice(ProviderId providerId,
+                                     Device oldDevice,
+                                     Device newDevice, Timestamp newTimestamp) {
+
+        // We allow only certain attributes to trigger update
+        if (!Objects.equals(oldDevice.hwVersion(), newDevice.hwVersion()) ||
+            !Objects.equals(oldDevice.swVersion(), newDevice.swVersion()) ||
+            !AnnotationsUtil.isEqual(oldDevice.annotations(), newDevice.annotations())) {
+
+            boolean replaced = devices.replace(newDevice.id(), oldDevice, newDevice);
+            if (!replaced) {
+                verify(replaced,
+                        "Replacing devices cache failed. PID:%s [expected:%s, found:%s, new=%s]",
+                        providerId, oldDevice, devices.get(newDevice.id())
+                        , newDevice);
+            }
+            if (!providerId.isAncillary()) {
+                markOnline(newDevice.id(), newTimestamp);
+            }
+            return new DeviceEvent(DeviceEvent.Type.DEVICE_UPDATED, newDevice, null);
+        }
+
+        // Otherwise merely attempt to change availability if primary provider
+        if (!providerId.isAncillary()) {
+            boolean added = markOnline(newDevice.id(), newTimestamp);
+            return !added ? null :
+                    new DeviceEvent(DEVICE_AVAILABILITY_CHANGED, newDevice, null);
+        }
+        return null;
+    }
+
+    @Override
+    public DeviceEvent markOffline(DeviceId deviceId) {
+        Timestamp timestamp = clockService.getTimestamp(deviceId);
+        DeviceEvent event = markOfflineInternal(deviceId, timestamp);
+        if (event != null) {
+            log.info("Notifying peers of a device offline topology event for deviceId: {}",
+                    deviceId);
+            try {
+                notifyPeers(new InternalDeviceOfflineEvent(deviceId, timestamp));
+            } catch (IOException e) {
+                log.error("Failed to notify peers of a device offline topology event for deviceId: {}",
+                     deviceId);
+            }
+        }
+        return event;
+    }
+
+    private DeviceEvent markOfflineInternal(DeviceId deviceId, Timestamp timestamp) {
+
+        Map<ProviderId, DeviceDescriptions> providerDescs
+            = getDeviceDescriptions(deviceId);
+
+        // locking device
+        synchronized (providerDescs) {
+
+            // accept off-line if given timestamp is newer than
+            // the latest Timestamp from Primary provider
+            DeviceDescriptions primDescs = getPrimaryDescriptions(providerDescs);
+            Timestamp lastTimestamp = primDescs.getLatestTimestamp();
+            if (timestamp.compareTo(lastTimestamp) <= 0) {
+                // outdated event ignore
+                return null;
+            }
+
+            offline.put(deviceId, timestamp);
+
+            Device device = devices.get(deviceId);
+            if (device == null) {
+                return null;
+            }
+            boolean removed = availableDevices.remove(deviceId);
+            if (removed) {
+                // TODO: broadcast ... DOWN only?
+                return new DeviceEvent(DEVICE_AVAILABILITY_CHANGED, device, null);
+            }
+            return null;
+        }
+    }
+
+    /**
+     * Marks the device as available if the given timestamp is not outdated,
+     * compared to the time the device has been marked offline.
+     *
+     * @param deviceId identifier of the device
+     * @param timestamp of the event triggering this change.
+     * @return true if availability change request was accepted and changed the state
+     */
+    // Guarded by deviceDescs value (=Device lock)
+    private boolean markOnline(DeviceId deviceId, Timestamp timestamp) {
+        // accept on-line if given timestamp is newer than
+        // the latest offline request Timestamp
+        Timestamp offlineTimestamp = offline.get(deviceId);
+        if (offlineTimestamp == null ||
+            offlineTimestamp.compareTo(timestamp) < 0) {
+
+            offline.remove(deviceId);
+            return availableDevices.add(deviceId);
+        }
+        return false;
+    }
+
+    @Override
+    public synchronized List<DeviceEvent> updatePorts(ProviderId providerId,
+                                       DeviceId deviceId,
+                                       List<PortDescription> portDescriptions) {
+        Timestamp newTimestamp = clockService.getTimestamp(deviceId);
+
+        Timestamped<List<PortDescription>> timestampedPortDescriptions =
+            new Timestamped<>(portDescriptions, newTimestamp);
+
+        List<DeviceEvent> events = updatePortsInternal(providerId, deviceId, timestampedPortDescriptions);
+        if (!events.isEmpty()) {
+            log.info("Notifying peers of a port update topology event for providerId: {} and deviceId: {}",
+                    providerId, deviceId);
+            try {
+                notifyPeers(new InternalPortEvent(providerId, deviceId, timestampedPortDescriptions));
+            } catch (IOException e) {
+                log.error("Failed to notify peers of a port update topology event or providerId: "
+                    + providerId + " and deviceId: " + deviceId, e);
+            }
+        }
+        return events;
+    }
+
+    private List<DeviceEvent> updatePortsInternal(ProviderId providerId,
+                                DeviceId deviceId,
+                                Timestamped<List<PortDescription>> portDescriptions) {
+
+        Device device = devices.get(deviceId);
+        checkArgument(device != null, DEVICE_NOT_FOUND, deviceId);
+
+        ConcurrentMap<ProviderId, DeviceDescriptions> descsMap = deviceDescs.get(deviceId);
+        checkArgument(descsMap != null, DEVICE_NOT_FOUND, deviceId);
+
+        List<DeviceEvent> events = new ArrayList<>();
+        synchronized (descsMap) {
+
+            if (isDeviceRemoved(deviceId, portDescriptions.timestamp())) {
+                log.debug("Ignoring outdated events: {}", portDescriptions);
+                return null;
+            }
+
+            DeviceDescriptions descs = descsMap.get(providerId);
+            // every provider must provide DeviceDescription.
+            checkArgument(descs != null,
+                    "Device description for Device ID %s from Provider %s was not found",
+                    deviceId, providerId);
+
+            Map<PortNumber, Port> ports = getPortMap(deviceId);
+
+            final Timestamp newTimestamp = portDescriptions.timestamp();
+
+            // Add new ports
+            Set<PortNumber> processed = new HashSet<>();
+            for (PortDescription portDescription : portDescriptions.value()) {
+                final PortNumber number = portDescription.portNumber();
+                processed.add(number);
+
+                final Port oldPort = ports.get(number);
+                final Port newPort;
+
+
+                final Timestamped<PortDescription> existingPortDesc = descs.getPortDesc(number);
+                if (existingPortDesc == null ||
+                    newTimestamp.compareTo(existingPortDesc.timestamp()) >= 0) {
+                    // on new port or valid update
+                    // update description
+                    descs.putPortDesc(new Timestamped<>(portDescription,
+                                            portDescriptions.timestamp()));
+                    newPort = composePort(device, number, descsMap);
+                } else {
+                    // outdated event, ignored.
+                    continue;
+                }
+
+                events.add(oldPort == null ?
+                                   createPort(device, newPort, ports) :
+                                   updatePort(device, oldPort, newPort, ports));
+            }
+
+            events.addAll(pruneOldPorts(device, ports, processed));
+        }
+        return FluentIterable.from(events).filter(notNull()).toList();
+    }
+
+    // Creates a new port based on the port description adds it to the map and
+    // Returns corresponding event.
+    // Guarded by deviceDescs value (=Device lock)
+    private DeviceEvent createPort(Device device, Port newPort,
+                                   Map<PortNumber, Port> ports) {
+        ports.put(newPort.number(), newPort);
+        return new DeviceEvent(PORT_ADDED, device, newPort);
+    }
+
+    // Checks if the specified port requires update and if so, it replaces the
+    // existing entry in the map and returns corresponding event.
+    // Guarded by deviceDescs value (=Device lock)
+    private DeviceEvent updatePort(Device device, Port oldPort,
+                                   Port newPort,
+                                   Map<PortNumber, Port> ports) {
+        if (oldPort.isEnabled() != newPort.isEnabled() ||
+            !AnnotationsUtil.isEqual(oldPort.annotations(), newPort.annotations())) {
+
+            ports.put(oldPort.number(), newPort);
+            return new DeviceEvent(PORT_UPDATED, device, newPort);
+        }
+        return null;
+    }
+
+    // Prunes the specified list of ports based on which ports are in the
+    // processed list and returns list of corresponding events.
+    // Guarded by deviceDescs value (=Device lock)
+    private List<DeviceEvent> pruneOldPorts(Device device,
+                                            Map<PortNumber, Port> ports,
+                                            Set<PortNumber> processed) {
+        List<DeviceEvent> events = new ArrayList<>();
+        Iterator<PortNumber> iterator = ports.keySet().iterator();
+        while (iterator.hasNext()) {
+            PortNumber portNumber = iterator.next();
+            if (!processed.contains(portNumber)) {
+                events.add(new DeviceEvent(PORT_REMOVED, device,
+                                           ports.get(portNumber)));
+                iterator.remove();
+            }
+        }
+        return events;
+    }
+
+    // Gets the map of ports for the specified device; if one does not already
+    // exist, it creates and registers a new one.
+    private ConcurrentMap<PortNumber, Port> getPortMap(DeviceId deviceId) {
+        return createIfAbsentUnchecked(devicePorts, deviceId,
+                NewConcurrentHashMap.<PortNumber, Port>ifNeeded());
+    }
+
+    private ConcurrentMap<ProviderId, DeviceDescriptions> getDeviceDescriptions(
+            DeviceId deviceId) {
+        return createIfAbsentUnchecked(deviceDescs, deviceId,
+                NewConcurrentHashMap.<ProviderId, DeviceDescriptions>ifNeeded());
+    }
+
+    @Override
+    public synchronized DeviceEvent updatePortStatus(ProviderId providerId, DeviceId deviceId,
+            PortDescription portDescription) {
+        Timestamp newTimestamp = clockService.getTimestamp(deviceId);
+        final Timestamped<PortDescription> deltaDesc = new Timestamped<>(portDescription, newTimestamp);
+        DeviceEvent event = updatePortStatusInternal(providerId, deviceId, deltaDesc);
+        if (event != null) {
+            log.info("Notifying peers of a port status update topology event for providerId: {} and deviceId: {}",
+                        providerId, deviceId);
+            try {
+                notifyPeers(new InternalPortStatusEvent(providerId, deviceId, deltaDesc));
+            } catch (IOException e) {
+                log.error("Failed to notify peers of a port status update topology event or providerId: "
+                        + providerId + " and deviceId: " + deviceId, e);
+            }
+        }
+        return event;
+    }
+
+    private DeviceEvent updatePortStatusInternal(ProviderId providerId, DeviceId deviceId,
+                Timestamped<PortDescription> deltaDesc) {
+
+        Device device = devices.get(deviceId);
+        checkArgument(device != null, DEVICE_NOT_FOUND, deviceId);
+
+        ConcurrentMap<ProviderId, DeviceDescriptions> descsMap = deviceDescs.get(deviceId);
+        checkArgument(descsMap != null, DEVICE_NOT_FOUND, deviceId);
+
+        synchronized (descsMap) {
+
+            if (isDeviceRemoved(deviceId, deltaDesc.timestamp())) {
+                log.debug("Ignoring outdated event: {}", deltaDesc);
+                return null;
+            }
+
+            DeviceDescriptions descs = descsMap.get(providerId);
+            // assuming all providers must to give DeviceDescription
+            checkArgument(descs != null,
+                    "Device description for Device ID %s from Provider %s was not found",
+                    deviceId, providerId);
+
+            ConcurrentMap<PortNumber, Port> ports = getPortMap(deviceId);
+            final PortNumber number = deltaDesc.value().portNumber();
+            final Port oldPort = ports.get(number);
+            final Port newPort;
+
+            final Timestamped<PortDescription> existingPortDesc = descs.getPortDesc(number);
+            if (existingPortDesc == null ||
+                deltaDesc == existingPortDesc ||
+                deltaDesc.isNewer(existingPortDesc)) {
+                // on new port or valid update
+                // update description
+                descs.putPortDesc(deltaDesc);
+                newPort = composePort(device, number, descsMap);
+            } else {
+                // outdated event, ignored.
+                return null;
+            }
+
+            if (oldPort == null) {
+                return createPort(device, newPort, ports);
+            } else {
+                return updatePort(device, oldPort, newPort, ports);
+            }
+        }
+    }
+
+    @Override
+    public List<Port> getPorts(DeviceId deviceId) {
+        Map<PortNumber, Port> ports = devicePorts.get(deviceId);
+        if (ports == null) {
+            return Collections.emptyList();
+        }
+        return ImmutableList.copyOf(ports.values());
+    }
+
+    @Override
+    public Port getPort(DeviceId deviceId, PortNumber portNumber) {
+        Map<PortNumber, Port> ports = devicePorts.get(deviceId);
+        return ports == null ? null : ports.get(portNumber);
+    }
+
+    @Override
+    public boolean isAvailable(DeviceId deviceId) {
+        return availableDevices.contains(deviceId);
+    }
+
+    @Override
+    public synchronized DeviceEvent removeDevice(DeviceId deviceId) {
+        Timestamp timestamp = clockService.getTimestamp(deviceId);
+        DeviceEvent event = removeDeviceInternal(deviceId, timestamp);
+        if (event != null) {
+            log.info("Notifying peers of a device removed topology event for deviceId: {}",
+                    deviceId);
+            try {
+                notifyPeers(new InternalDeviceRemovedEvent(deviceId, timestamp));
+            } catch (IOException e) {
+                log.error("Failed to notify peers of a device removed topology event for deviceId: {}",
+                     deviceId);
+            }
+        }
+        return event;
+    }
+
+    private DeviceEvent removeDeviceInternal(DeviceId deviceId,
+                                             Timestamp timestamp) {
+
+        Map<ProviderId, DeviceDescriptions> descs = getDeviceDescriptions(deviceId);
+        synchronized (descs) {
+            // accept removal request if given timestamp is newer than
+            // the latest Timestamp from Primary provider
+            DeviceDescriptions primDescs = getPrimaryDescriptions(descs);
+            Timestamp lastTimestamp = primDescs.getLatestTimestamp();
+            if (timestamp.compareTo(lastTimestamp) <= 0) {
+                // outdated event ignore
+                return null;
+            }
+            removalRequest.put(deviceId, timestamp);
+
+            Device device = devices.remove(deviceId);
+            // should DEVICE_REMOVED carry removed ports?
+            Map<PortNumber, Port> ports = devicePorts.get(deviceId);
+            if (ports != null) {
+                ports.clear();
+            }
+            markOfflineInternal(deviceId, timestamp);
+            descs.clear();
+            return device == null ? null :
+                new DeviceEvent(DEVICE_REMOVED, device, null);
+        }
+    }
+
+    private boolean isDeviceRemoved(DeviceId deviceId, Timestamp timestampToCheck) {
+        Timestamp removalTimestamp = removalRequest.get(deviceId);
+        if (removalTimestamp != null &&
+            removalTimestamp.compareTo(timestampToCheck) >= 0) {
+            // removalRequest is more recent
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Returns a Device, merging description given from multiple Providers.
+     *
+     * @param deviceId device identifier
+     * @param providerDescs Collection of Descriptions from multiple providers
+     * @return Device instance
+     */
+    private Device composeDevice(DeviceId deviceId,
+            ConcurrentMap<ProviderId, DeviceDescriptions> providerDescs) {
+
+        checkArgument(!providerDescs.isEmpty(), "No Device descriptions supplied");
+
+        ProviderId primary = pickPrimaryPID(providerDescs);
+
+        DeviceDescriptions desc = providerDescs.get(primary);
+
+        final DeviceDescription base = desc.getDeviceDesc().value();
+        Type type = base.type();
+        String manufacturer = base.manufacturer();
+        String hwVersion = base.hwVersion();
+        String swVersion = base.swVersion();
+        String serialNumber = base.serialNumber();
+        DefaultAnnotations annotations = DefaultAnnotations.builder().build();
+        annotations = merge(annotations, base.annotations());
+
+        for (Entry<ProviderId, DeviceDescriptions> e : providerDescs.entrySet()) {
+            if (e.getKey().equals(primary)) {
+                continue;
+            }
+            // TODO: should keep track of Description timestamp
+            // and only merge conflicting keys when timestamp is newer
+            // Currently assuming there will never be a key conflict between
+            // providers
+
+            // annotation merging. not so efficient, should revisit later
+            annotations = merge(annotations, e.getValue().getDeviceDesc().value().annotations());
+        }
+
+        return new DefaultDevice(primary, deviceId , type, manufacturer,
+                            hwVersion, swVersion, serialNumber, annotations);
+    }
+
+    /**
+     * Returns a Port, merging description given from multiple Providers.
+     *
+     * @param device device the port is on
+     * @param number port number
+     * @param providerDescs Collection of Descriptions from multiple providers
+     * @return Port instance
+     */
+    private Port composePort(Device device, PortNumber number,
+                ConcurrentMap<ProviderId, DeviceDescriptions> providerDescs) {
+
+        ProviderId primary = pickPrimaryPID(providerDescs);
+        DeviceDescriptions primDescs = providerDescs.get(primary);
+        // if no primary, assume not enabled
+        // TODO: revisit this default port enabled/disabled behavior
+        boolean isEnabled = false;
+        DefaultAnnotations annotations = DefaultAnnotations.builder().build();
+
+        final Timestamped<PortDescription> portDesc = primDescs.getPortDesc(number);
+        if (portDesc != null) {
+            isEnabled = portDesc.value().isEnabled();
+            annotations = merge(annotations, portDesc.value().annotations());
+        }
+
+        for (Entry<ProviderId, DeviceDescriptions> e : providerDescs.entrySet()) {
+            if (e.getKey().equals(primary)) {
+                continue;
+            }
+            // TODO: should keep track of Description timestamp
+            // and only merge conflicting keys when timestamp is newer
+            // Currently assuming there will never be a key conflict between
+            // providers
+
+            // annotation merging. not so efficient, should revisit later
+            final Timestamped<PortDescription> otherPortDesc = e.getValue().getPortDesc(number);
+            if (otherPortDesc != null) {
+                annotations = merge(annotations, otherPortDesc.value().annotations());
+            }
+        }
+
+        return new DefaultPort(device, number, isEnabled, annotations);
+    }
+
+    /**
+     * @return primary ProviderID, or randomly chosen one if none exists
+     */
+    private ProviderId pickPrimaryPID(
+            Map<ProviderId, DeviceDescriptions> providerDescs) {
+        ProviderId fallBackPrimary = null;
+        for (Entry<ProviderId, DeviceDescriptions> e : providerDescs.entrySet()) {
+            if (!e.getKey().isAncillary()) {
+                return e.getKey();
+            } else if (fallBackPrimary == null) {
+                // pick randomly as a fallback in case there is no primary
+                fallBackPrimary = e.getKey();
+            }
+        }
+        return fallBackPrimary;
+    }
+
+    private DeviceDescriptions getPrimaryDescriptions(
+                            Map<ProviderId, DeviceDescriptions> providerDescs) {
+        ProviderId pid = pickPrimaryPID(providerDescs);
+        return providerDescs.get(pid);
+    }
+
+    public static final class InitDeviceDescs
+        implements ConcurrentInitializer<DeviceDescriptions> {
+
+        private final Timestamped<DeviceDescription> deviceDesc;
+
+        public InitDeviceDescs(Timestamped<DeviceDescription> deviceDesc) {
+            this.deviceDesc = checkNotNull(deviceDesc);
+        }
+        @Override
+        public DeviceDescriptions get() throws ConcurrentException {
+            return new DeviceDescriptions(deviceDesc);
+        }
+    }
+
+
+    /**
+     * Collection of Description of a Device and it's Ports given from a Provider.
+     */
+    public static class DeviceDescriptions {
+
+        private final AtomicReference<Timestamped<DeviceDescription>> deviceDesc;
+        private final ConcurrentMap<PortNumber, Timestamped<PortDescription>> portDescs;
+
+        public DeviceDescriptions(Timestamped<DeviceDescription> desc) {
+            this.deviceDesc = new AtomicReference<>(checkNotNull(desc));
+            this.portDescs = new ConcurrentHashMap<>();
+        }
+
+        Timestamp getLatestTimestamp() {
+            Timestamp latest = deviceDesc.get().timestamp();
+            for (Timestamped<PortDescription> desc : portDescs.values()) {
+                if (desc.timestamp().compareTo(latest) > 0) {
+                    latest = desc.timestamp();
+                }
+            }
+            return latest;
+        }
+
+        public Timestamped<DeviceDescription> getDeviceDesc() {
+            return deviceDesc.get();
+        }
+
+        public Timestamped<PortDescription> getPortDesc(PortNumber number) {
+            return portDescs.get(number);
+        }
+
+        /**
+         * Puts DeviceDescription, merging annotations as necessary.
+         *
+         * @param newDesc new DeviceDescription
+         * @return previous DeviceDescription
+         */
+        public synchronized Timestamped<DeviceDescription> putDeviceDesc(Timestamped<DeviceDescription> newDesc) {
+            Timestamped<DeviceDescription> oldOne = deviceDesc.get();
+            Timestamped<DeviceDescription> newOne = newDesc;
+            if (oldOne != null) {
+                SparseAnnotations merged = union(oldOne.value().annotations(),
+                                                 newDesc.value().annotations());
+                newOne = new Timestamped<DeviceDescription>(
+                        new DefaultDeviceDescription(newDesc.value(), merged),
+                        newDesc.timestamp());
+            }
+            return deviceDesc.getAndSet(newOne);
+        }
+
+        /**
+         * Puts PortDescription, merging annotations as necessary.
+         *
+         * @param newDesc new PortDescription
+         * @return previous PortDescription
+         */
+        public synchronized Timestamped<PortDescription> putPortDesc(Timestamped<PortDescription> newDesc) {
+            Timestamped<PortDescription> oldOne = portDescs.get(newDesc.value().portNumber());
+            Timestamped<PortDescription> newOne = newDesc;
+            if (oldOne != null) {
+                SparseAnnotations merged = union(oldOne.value().annotations(),
+                                                 newDesc.value().annotations());
+                newOne = new Timestamped<PortDescription>(
+                        new DefaultPortDescription(newDesc.value(), merged),
+                        newDesc.timestamp());
+            }
+            return portDescs.put(newOne.value().portNumber(), newOne);
+        }
+    }
+
+    private void notifyPeers(InternalDeviceEvent event) throws IOException {
+        ClusterMessage message = new ClusterMessage(
+                clusterService.getLocalNode().id(),
+                GossipDeviceStoreMessageSubjects.DEVICE_UPDATE,
+                SERIALIZER.encode(event));
+        clusterCommunicator.broadcast(message);
+    }
+
+    private void notifyPeers(InternalDeviceOfflineEvent event) throws IOException {
+        ClusterMessage message = new ClusterMessage(
+                clusterService.getLocalNode().id(),
+                GossipDeviceStoreMessageSubjects.DEVICE_OFFLINE,
+                SERIALIZER.encode(event));
+        clusterCommunicator.broadcast(message);
+    }
+
+    private void notifyPeers(InternalDeviceRemovedEvent event) throws IOException {
+        ClusterMessage message = new ClusterMessage(
+                clusterService.getLocalNode().id(),
+                GossipDeviceStoreMessageSubjects.DEVICE_REMOVED,
+                SERIALIZER.encode(event));
+        clusterCommunicator.broadcast(message);
+    }
+
+    private void notifyPeers(InternalPortEvent event) throws IOException {
+        ClusterMessage message = new ClusterMessage(
+                clusterService.getLocalNode().id(),
+                GossipDeviceStoreMessageSubjects.PORT_UPDATE,
+                SERIALIZER.encode(event));
+        clusterCommunicator.broadcast(message);
+    }
+
+    private void notifyPeers(InternalPortStatusEvent event) throws IOException {
+        ClusterMessage message = new ClusterMessage(
+                clusterService.getLocalNode().id(),
+                GossipDeviceStoreMessageSubjects.PORT_STATUS_UPDATE,
+                SERIALIZER.encode(event));
+        clusterCommunicator.broadcast(message);
+    }
+
+    private class InternalDeviceEventListener implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.info("Received device update event from peer: {}", message.sender());
+            InternalDeviceEvent event = (InternalDeviceEvent) SERIALIZER.decode(message.payload());
+
+            ProviderId providerId = event.providerId();
+            DeviceId deviceId = event.deviceId();
+            Timestamped<DeviceDescription> deviceDescription = event.deviceDescription();
+
+            createOrUpdateDeviceInternal(providerId, deviceId, deviceDescription);
+        }
+    }
+
+    private class InternalDeviceOfflineEventListener implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.info("Received device offline event from peer: {}", message.sender());
+            InternalDeviceOfflineEvent event = (InternalDeviceOfflineEvent) SERIALIZER.decode(message.payload());
+
+            DeviceId deviceId = event.deviceId();
+            Timestamp timestamp = event.timestamp();
+
+            markOfflineInternal(deviceId, timestamp);
+        }
+    }
+
+    private class InternalDeviceRemovedEventListener implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.info("Received device removed event from peer: {}", message.sender());
+            InternalDeviceRemovedEvent event = (InternalDeviceRemovedEvent) SERIALIZER.decode(message.payload());
+
+            DeviceId deviceId = event.deviceId();
+            Timestamp timestamp = event.timestamp();
+
+            removeDeviceInternal(deviceId, timestamp);
+        }
+    }
+
+    private class InternalPortEventListener implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.info("Received port update event from peer: {}", message.sender());
+            InternalPortEvent event = (InternalPortEvent) SERIALIZER.decode(message.payload());
+
+            ProviderId providerId = event.providerId();
+            DeviceId deviceId = event.deviceId();
+            Timestamped<List<PortDescription>> portDescriptions = event.portDescriptions();
+
+            updatePortsInternal(providerId, deviceId, portDescriptions);
+        }
+    }
+
+    private class InternalPortStatusEventListener implements ClusterMessageHandler {
+        @Override
+        public void handle(ClusterMessage message) {
+
+            log.info("Received port status update event from peer: {}", message.sender());
+            InternalPortStatusEvent event = (InternalPortStatusEvent) SERIALIZER.decode(message.payload());
+
+            ProviderId providerId = event.providerId();
+            DeviceId deviceId = event.deviceId();
+            Timestamped<PortDescription> portDescription = event.portDescription();
+
+            updatePortStatusInternal(providerId, deviceId, portDescription);
+        }
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
+
+/**
+ * MessageSubjects used by GossipDeviceStore peer-peer communication.
+ */
+public final class GossipDeviceStoreMessageSubjects {
+
+    private GossipDeviceStoreMessageSubjects() {}
+
+    public static final MessageSubject DEVICE_UPDATE = new MessageSubject("peer-device-update");
+    public static final MessageSubject DEVICE_OFFLINE = new MessageSubject("peer-device-offline");
+    public static final MessageSubject DEVICE_REMOVED = new MessageSubject("peer-device-removed");
+    public static final MessageSubject PORT_UPDATE = new MessageSubject("peer-port-update");
+    public static final MessageSubject PORT_STATUS_UPDATE = new MessageSubject("peer-port-status-update");
+}

+package org.onlab.onos.store.device.impl;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.device.DeviceDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+/**
+ * Information published by GossipDeviceStore to notify peers of a device
+ * change event.
+ */
+public class InternalDeviceEvent {
+
+    private final ProviderId providerId;
+    private final DeviceId deviceId;
+    private final Timestamped<DeviceDescription> deviceDescription;
+
+    protected InternalDeviceEvent(
+            ProviderId providerId,
+            DeviceId deviceId,
+            Timestamped<DeviceDescription> deviceDescription) {
+        this.providerId = providerId;
+        this.deviceId = deviceId;
+        this.deviceDescription = deviceDescription;
+    }
+
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    public ProviderId providerId() {
+        return providerId;
+    }
+
+    public Timestamped<DeviceDescription> deviceDescription() {
+        return deviceDescription;
+    }
+
+    // for serializer
+    protected InternalDeviceEvent() {
+        this.providerId = null;
+        this.deviceId = null;
+        this.deviceDescription = null;
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.device.DeviceDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Kryo Serializer for {@link InternalDeviceEvent}.
+ */
+public class InternalDeviceEventSerializer extends Serializer<InternalDeviceEvent> {
+
+    /**
+     * Creates a serializer for {@link InternalDeviceEvent}.
+     */
+    public InternalDeviceEventSerializer() {
+        // does not accept null
+        super(false);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output, InternalDeviceEvent event) {
+        kryo.writeClassAndObject(output, event.providerId());
+        kryo.writeClassAndObject(output, event.deviceId());
+        kryo.writeClassAndObject(output, event.deviceDescription());
+    }
+
+    @Override
+    public InternalDeviceEvent read(Kryo kryo, Input input,
+                               Class<InternalDeviceEvent> type) {
+        ProviderId providerId = (ProviderId) kryo.readClassAndObject(input);
+        DeviceId deviceId = (DeviceId) kryo.readClassAndObject(input);
+        Timestamped<DeviceDescription> deviceDescription
+            = (Timestamped<DeviceDescription>) kryo.readClassAndObject(input);
+
+        return new InternalDeviceEvent(providerId, deviceId, deviceDescription);
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.store.Timestamp;
+
+/**
+ * Information published by GossipDeviceStore to notify peers of a device
+ * going offline.
+ */
+public class InternalDeviceOfflineEvent {
+
+    private final DeviceId deviceId;
+    private final Timestamp timestamp;
+
+    /**
+     * Creates a InternalDeviceOfflineEvent.
+     * @param deviceId identifier of device going offline.
+     * @param timestamp timestamp of when the device went offline.
+     */
+    public InternalDeviceOfflineEvent(DeviceId deviceId, Timestamp timestamp) {
+        this.deviceId = deviceId;
+        this.timestamp = timestamp;
+    }
+
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    public Timestamp timestamp() {
+        return timestamp;
+    }
+
+    // for serializer
+    @SuppressWarnings("unused")
+    private InternalDeviceOfflineEvent() {
+        deviceId = null;
+        timestamp = null;
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.store.Timestamp;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Kryo Serializer for {@link InternalDeviceOfflineEvent}.
+ */
+public class InternalDeviceOfflineEventSerializer extends Serializer<InternalDeviceOfflineEvent> {
+
+    /**
+     * Creates a serializer for {@link InternalDeviceOfflineEvent}.
+     */
+    public InternalDeviceOfflineEventSerializer() {
+        // does not accept null
+        super(false);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output, InternalDeviceOfflineEvent event) {
+        kryo.writeClassAndObject(output, event.deviceId());
+        kryo.writeClassAndObject(output, event.timestamp());
+    }
+
+    @Override
+    public InternalDeviceOfflineEvent read(Kryo kryo, Input input,
+                               Class<InternalDeviceOfflineEvent> type) {
+        DeviceId deviceId = (DeviceId) kryo.readClassAndObject(input);
+        Timestamp timestamp = (Timestamp) kryo.readClassAndObject(input);
+
+        return new InternalDeviceOfflineEvent(deviceId, timestamp);
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.store.Timestamp;
+
+/**
+ * Information published by GossipDeviceStore to notify peers of a device
+ * being administratively removed.
+ */
+public class InternalDeviceRemovedEvent {
+
+    private final DeviceId deviceId;
+    private final Timestamp timestamp;
+
+    /**
+     * Creates a InternalDeviceRemovedEvent.
+     * @param deviceId identifier of the removed device.
+     * @param timestamp timestamp of when the device was administratively removed.
+     */
+    public InternalDeviceRemovedEvent(DeviceId deviceId, Timestamp timestamp) {
+        this.deviceId = deviceId;
+        this.timestamp = timestamp;
+    }
+
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    public Timestamp timestamp() {
+        return timestamp;
+    }
+
+    // for serializer
+    @SuppressWarnings("unused")
+    private InternalDeviceRemovedEvent() {
+        deviceId = null;
+        timestamp = null;
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import java.util.List;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.device.PortDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+/**
+ * Information published by GossipDeviceStore to notify peers of a port
+ * change event.
+ */
+public class InternalPortEvent {
+
+    private final ProviderId providerId;
+    private final DeviceId deviceId;
+    private final Timestamped<List<PortDescription>> portDescriptions;
+
+    protected InternalPortEvent(
+            ProviderId providerId,
+            DeviceId deviceId,
+            Timestamped<List<PortDescription>> portDescriptions) {
+        this.providerId = providerId;
+        this.deviceId = deviceId;
+        this.portDescriptions = portDescriptions;
+    }
+
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    public ProviderId providerId() {
+        return providerId;
+    }
+
+    public Timestamped<List<PortDescription>> portDescriptions() {
+        return portDescriptions;
+    }
+
+    // for serializer
+    protected InternalPortEvent() {
+        this.providerId = null;
+        this.deviceId = null;
+        this.portDescriptions = null;
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import java.util.List;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.device.PortDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Kryo Serializer for {@link InternalPortEvent}.
+ */
+public class InternalPortEventSerializer extends Serializer<InternalPortEvent> {
+
+    /**
+     * Creates a serializer for {@link InternalPortEvent}.
+     */
+    public InternalPortEventSerializer() {
+        // does not accept null
+        super(false);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output, InternalPortEvent event) {
+        kryo.writeClassAndObject(output, event.providerId());
+        kryo.writeClassAndObject(output, event.deviceId());
+        kryo.writeClassAndObject(output, event.portDescriptions());
+    }
+
+    @Override
+    public InternalPortEvent read(Kryo kryo, Input input,
+                               Class<InternalPortEvent> type) {
+        ProviderId providerId = (ProviderId) kryo.readClassAndObject(input);
+        DeviceId deviceId = (DeviceId) kryo.readClassAndObject(input);
+        Timestamped<List<PortDescription>> portDescriptions
+            = (Timestamped<List<PortDescription>>) kryo.readClassAndObject(input);
+
+        return new InternalPortEvent(providerId, deviceId, portDescriptions);
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.device.PortDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+/**
+ * Information published by GossipDeviceStore to notify peers of a port
+ * status change event.
+ */
+public class InternalPortStatusEvent {
+
+    private final ProviderId providerId;
+    private final DeviceId deviceId;
+    private final Timestamped<PortDescription> portDescription;
+
+    protected InternalPortStatusEvent(
+            ProviderId providerId,
+            DeviceId deviceId,
+            Timestamped<PortDescription> portDescription) {
+        this.providerId = providerId;
+        this.deviceId = deviceId;
+        this.portDescription = portDescription;
+    }
+
+    public DeviceId deviceId() {
+        return deviceId;
+    }
+
+    public ProviderId providerId() {
+        return providerId;
+    }
+
+    public Timestamped<PortDescription> portDescription() {
+        return portDescription;
+    }
+
+    // for serializer
+    protected InternalPortStatusEvent() {
+        this.providerId = null;
+        this.deviceId = null;
+        this.portDescription = null;
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.device.PortDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.common.impl.Timestamped;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Kryo Serializer for {@link InternalPortStatusEvent}.
+ */
+public class InternalPortStatusEventSerializer extends Serializer<InternalPortStatusEvent> {
+
+    /**
+     * Creates a serializer for {@link InternalPortStatusEvent}.
+     */
+    public InternalPortStatusEventSerializer() {
+        // does not accept null
+        super(false);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output, InternalPortStatusEvent event) {
+        kryo.writeClassAndObject(output, event.providerId());
+        kryo.writeClassAndObject(output, event.deviceId());
+        kryo.writeClassAndObject(output, event.portDescription());
+    }
+
+    @Override
+    public InternalPortStatusEvent read(Kryo kryo, Input input,
+                               Class<InternalPortStatusEvent> type) {
+        ProviderId providerId = (ProviderId) kryo.readClassAndObject(input);
+        DeviceId deviceId = (DeviceId) kryo.readClassAndObject(input);
+        Timestamped<PortDescription> portDescription = (Timestamped<PortDescription>) kryo.readClassAndObject(input);
+
+        return new InternalPortStatusEvent(providerId, deviceId, portDescription);
+    }
+}

-package org.onlab.onos.store.device.impl;
-
-import static com.google.common.base.Predicates.notNull;
-import static com.google.common.base.Preconditions.checkState;
-
-import com.google.common.collect.FluentIterable;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.ImmutableSet.Builder;
-
-import org.apache.felix.scr.annotations.Activate;
-import org.apache.felix.scr.annotations.Component;
-import org.apache.felix.scr.annotations.Deactivate;
-import org.apache.felix.scr.annotations.Reference;
-import org.apache.felix.scr.annotations.ReferenceCardinality;
-import org.apache.felix.scr.annotations.Service;
-import org.onlab.onos.net.DefaultDevice;
-import org.onlab.onos.net.DefaultPort;
-import org.onlab.onos.net.Device;
-import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.net.Port;
-import org.onlab.onos.net.PortNumber;
-import org.onlab.onos.net.device.DeviceDescription;
-import org.onlab.onos.net.device.DeviceEvent;
-import org.onlab.onos.net.device.DeviceStore;
-import org.onlab.onos.net.device.DeviceStoreDelegate;
-import org.onlab.onos.net.device.PortDescription;
-import org.onlab.onos.net.provider.ProviderId;
-import org.onlab.onos.store.AbstractStore;
-import org.onlab.onos.store.ClockService;
-import org.onlab.onos.store.Timestamp;
-import org.slf4j.Logger;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Objects;
-import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentMap;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static org.onlab.onos.net.device.DeviceEvent.Type.*;
-import static org.slf4j.LoggerFactory.getLogger;
-
-/**
- * Manages inventory of infrastructure devices using a protocol that takes into consideration
- * the order in which device events occur.
- */
-@Component(immediate = true)
-@Service
-public class OnosDistributedDeviceStore
-        extends AbstractStore<DeviceEvent, DeviceStoreDelegate>
-        implements DeviceStore {
-
-    private final Logger log = getLogger(getClass());
-
-    public static final String DEVICE_NOT_FOUND = "Device with ID %s not found";
-
-    private ConcurrentMap<DeviceId, VersionedValue<Device>> devices;
-    private ConcurrentMap<DeviceId, Map<PortNumber, VersionedValue<Port>>> devicePorts;
-
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected ClockService clockService;
-
-    @Activate
-    public void activate() {
-
-        devices = new ConcurrentHashMap<>();
-        devicePorts = new ConcurrentHashMap<>();
-
-        log.info("Started");
-    }
-
-    @Deactivate
-    public void deactivate() {
-        log.info("Stopped");
-    }
-
-    @Override
-    public int getDeviceCount() {
-        return devices.size();
-    }
-
-    @Override
-    public Iterable<Device> getDevices() {
-        Builder<Device> builder = ImmutableSet.builder();
-        synchronized (this) {
-            for (VersionedValue<Device> device : devices.values()) {
-                builder.add(device.entity());
-            }
-            return builder.build();
-        }
-    }
-
-    @Override
-    public Device getDevice(DeviceId deviceId) {
-        VersionedValue<Device> device = devices.get(deviceId);
-        checkArgument(device != null, DEVICE_NOT_FOUND, deviceId);
-        return device.entity();
-    }
-
-    @Override
-    public DeviceEvent createOrUpdateDevice(ProviderId providerId, DeviceId deviceId,
-                                            DeviceDescription deviceDescription) {
-        Timestamp newTimestamp = clockService.getTimestamp(deviceId);
-        VersionedValue<Device> device = devices.get(deviceId);
-
-        if (device == null) {
-            return createDevice(providerId, deviceId, deviceDescription, newTimestamp);
-        }
-
-        checkState(newTimestamp.compareTo(device.timestamp()) > 0,
-                "Existing device has a timestamp in the future!");
-
-        return updateDevice(providerId, device.entity(), deviceDescription, newTimestamp);
-    }
-
-    // Creates the device and returns the appropriate event if necessary.
-    private DeviceEvent createDevice(ProviderId providerId, DeviceId deviceId,
-                                     DeviceDescription desc, Timestamp timestamp) {
-        Device device = new DefaultDevice(providerId, deviceId, desc.type(),
-                                                 desc.manufacturer(),
-                                                 desc.hwVersion(), desc.swVersion(),
-                                                 desc.serialNumber());
-
-        devices.put(deviceId, new VersionedValue<>(device, true, timestamp));
-        // TODO,FIXME: broadcast a message telling peers of a device event.
-        return new DeviceEvent(DEVICE_ADDED, device, null);
-    }
-
-    // Updates the device and returns the appropriate event if necessary.
-    private DeviceEvent updateDevice(ProviderId providerId, Device device,
-                                     DeviceDescription desc, Timestamp timestamp) {
-        // We allow only certain attributes to trigger update
-        if (!Objects.equals(device.hwVersion(), desc.hwVersion()) ||
-                !Objects.equals(device.swVersion(), desc.swVersion())) {
-
-            Device updated = new DefaultDevice(providerId, device.id(),
-                                                      desc.type(),
-                                                      desc.manufacturer(),
-                                                      desc.hwVersion(),
-                                                      desc.swVersion(),
-                                                      desc.serialNumber());
-            devices.put(device.id(), new VersionedValue<Device>(updated, true, timestamp));
-            // FIXME: broadcast a message telling peers of a device event.
-            return new DeviceEvent(DeviceEvent.Type.DEVICE_UPDATED, updated, null);
-        }
-
-        // Otherwise merely attempt to change availability
-        Device updated = new DefaultDevice(providerId, device.id(),
-                desc.type(),
-                desc.manufacturer(),
-                desc.hwVersion(),
-                desc.swVersion(),
-                desc.serialNumber());
-
-        VersionedValue<Device> oldDevice = devices.put(device.id(),
-                new VersionedValue<Device>(updated, true, timestamp));
-        if (!oldDevice.isUp()) {
-            return new DeviceEvent(DEVICE_AVAILABILITY_CHANGED, device, null);
-        } else {
-            return null;
-        }
-    }
-
-    @Override
-    public DeviceEvent markOffline(DeviceId deviceId) {
-        VersionedValue<Device> device = devices.get(deviceId);
-        boolean willRemove = device != null && device.isUp();
-        if (!willRemove) {
-            return null;
-        }
-        Timestamp timestamp = clockService.getTimestamp(deviceId);
-        if (replaceIfLatest(device.entity(), false, timestamp)) {
-            return new DeviceEvent(DEVICE_AVAILABILITY_CHANGED, device.entity(), null);
-        }
-        return null;
-    }
-
-    // Replace existing value if its timestamp is older.
-    private synchronized boolean replaceIfLatest(Device device, boolean isUp, Timestamp timestamp) {
-        VersionedValue<Device> existingValue = devices.get(device.id());
-        if (timestamp.compareTo(existingValue.timestamp()) > 0) {
-            devices.put(device.id(), new VersionedValue<Device>(device, isUp, timestamp));
-            return true;
-        }
-        return false;
-    }
-
-    @Override
-    public List<DeviceEvent> updatePorts(ProviderId providerId, DeviceId deviceId,
-                                         List<PortDescription> portDescriptions) {
-        List<DeviceEvent> events = new ArrayList<>();
-        synchronized (this) {
-            VersionedValue<Device> device = devices.get(deviceId);
-            checkArgument(device != null, DEVICE_NOT_FOUND, deviceId);
-            Map<PortNumber, VersionedValue<Port>> ports = getPortMap(deviceId);
-            Timestamp newTimestamp = clockService.getTimestamp(deviceId);
-
-            // Add new ports
-            Set<PortNumber> processed = new HashSet<>();
-            for (PortDescription portDescription : portDescriptions) {
-                VersionedValue<Port> port = ports.get(portDescription.portNumber());
-                if (port == null) {
-                    events.add(createPort(device, portDescription, ports, newTimestamp));
-                }
-                checkState(newTimestamp.compareTo(port.timestamp()) > 0,
-                        "Existing port state has a timestamp in the future!");
-                events.add(updatePort(device.entity(), port.entity(), portDescription, ports, newTimestamp));
-                processed.add(portDescription.portNumber());
-            }
-
-            updatePortMap(deviceId, ports);
-
-            events.addAll(pruneOldPorts(device.entity(), ports, processed));
-        }
-        return FluentIterable.from(events).filter(notNull()).toList();
-    }
-
-    // Creates a new port based on the port description adds it to the map and
-    // Returns corresponding event.
-    //@GuardedBy("this")
-    private DeviceEvent createPort(VersionedValue<Device> device, PortDescription portDescription,
-                                   Map<PortNumber, VersionedValue<Port>> ports, Timestamp timestamp) {
-        Port port = new DefaultPort(device.entity(), portDescription.portNumber(),
-                                           portDescription.isEnabled());
-        ports.put(port.number(), new VersionedValue<Port>(port, true, timestamp));
-        updatePortMap(device.entity().id(), ports);
-        return new DeviceEvent(PORT_ADDED, device.entity(), port);
-    }
-
-    // Checks if the specified port requires update and if so, it replaces the
-    // existing entry in the map and returns corresponding event.
-    //@GuardedBy("this")
-    private DeviceEvent updatePort(Device device, Port port,
-                                   PortDescription portDescription,
-                                   Map<PortNumber, VersionedValue<Port>> ports,
-                                   Timestamp timestamp) {
-        if (port.isEnabled() != portDescription.isEnabled()) {
-            VersionedValue<Port> updatedPort = new VersionedValue<Port>(
-                    new DefaultPort(device, portDescription.portNumber(),
-                                    portDescription.isEnabled()),
-                    portDescription.isEnabled(),
-                    timestamp);
-            ports.put(port.number(), updatedPort);
-            updatePortMap(device.id(), ports);
-            return new DeviceEvent(PORT_UPDATED, device, updatedPort.entity());
-        }
-        return null;
-    }
-
-    // Prunes the specified list of ports based on which ports are in the
-    // processed list and returns list of corresponding events.
-    //@GuardedBy("this")
-    private List<DeviceEvent> pruneOldPorts(Device device,
-                                            Map<PortNumber, VersionedValue<Port>> ports,
-                                            Set<PortNumber> processed) {
-        List<DeviceEvent> events = new ArrayList<>();
-        Iterator<PortNumber> iterator = ports.keySet().iterator();
-        while (iterator.hasNext()) {
-            PortNumber portNumber = iterator.next();
-            if (!processed.contains(portNumber)) {
-                events.add(new DeviceEvent(PORT_REMOVED, device,
-                                           ports.get(portNumber).entity()));
-                iterator.remove();
-            }
-        }
-        if (!events.isEmpty()) {
-            updatePortMap(device.id(), ports);
-        }
-        return events;
-    }
-
-    // Gets the map of ports for the specified device; if one does not already
-    // exist, it creates and registers a new one.
-    // WARN: returned value is a copy, changes made to the Map
-    //       needs to be written back using updatePortMap
-    //@GuardedBy("this")
-    private Map<PortNumber, VersionedValue<Port>> getPortMap(DeviceId deviceId) {
-        Map<PortNumber, VersionedValue<Port>> ports = devicePorts.get(deviceId);
-        if (ports == null) {
-            ports = new HashMap<>();
-            // this probably is waste of time in most cases.
-            updatePortMap(deviceId, ports);
-        }
-        return ports;
-    }
-
-    //@GuardedBy("this")
-    private void updatePortMap(DeviceId deviceId, Map<PortNumber, VersionedValue<Port>> ports) {
-        devicePorts.put(deviceId, ports);
-    }
-
-    @Override
-    public DeviceEvent updatePortStatus(ProviderId providerId, DeviceId deviceId,
-                                        PortDescription portDescription) {
-        VersionedValue<Device> device = devices.get(deviceId);
-        checkArgument(device != null, DEVICE_NOT_FOUND, deviceId);
-        Map<PortNumber, VersionedValue<Port>> ports = getPortMap(deviceId);
-        VersionedValue<Port> port = ports.get(portDescription.portNumber());
-        Timestamp timestamp = clockService.getTimestamp(deviceId);
-        return updatePort(device.entity(), port.entity(), portDescription, ports, timestamp);
-    }
-
-    @Override
-    public List<Port> getPorts(DeviceId deviceId) {
-        Map<PortNumber, VersionedValue<Port>> versionedPorts = devicePorts.get(deviceId);
-        if (versionedPorts == null) {
-            return Collections.emptyList();
-        }
-        List<Port> ports = new ArrayList<>();
-        for (VersionedValue<Port> port : versionedPorts.values()) {
-            ports.add(port.entity());
-        }
-        return ports;
-    }
-
-    @Override
-    public Port getPort(DeviceId deviceId, PortNumber portNumber) {
-        Map<PortNumber, VersionedValue<Port>> ports = devicePorts.get(deviceId);
-        return ports == null ? null : ports.get(portNumber).entity();
-    }
-
-    @Override
-    public boolean isAvailable(DeviceId deviceId) {
-        return devices.get(deviceId).isUp();
-    }
-
-    @Override
-    public DeviceEvent removeDevice(DeviceId deviceId) {
-        VersionedValue<Device> previousDevice = devices.remove(deviceId);
-        return previousDevice == null ? null :
-            new DeviceEvent(DEVICE_REMOVED, previousDevice.entity(), null);
-    }
-}

 /**
  * Implementation of device store using distributed distributed p2p synchronization protocol.
  */
-package org.onlab.onos.store.device.impl;
\ No newline at end of file
+package org.onlab.onos.store.device.impl;

 package org.onlab.onos.store.flow.impl;
 
-import static org.onlab.onos.net.flow.FlowRuleEvent.Type.RULE_ADDED;
 import static org.onlab.onos.net.flow.FlowRuleEvent.Type.RULE_REMOVED;
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -13,9 +12,10 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.net.flow.DefaultFlowRule;
+import org.onlab.onos.net.flow.DefaultFlowEntry;
+import org.onlab.onos.net.flow.FlowEntry;
+import org.onlab.onos.net.flow.FlowEntry.FlowEntryState;
 import org.onlab.onos.net.flow.FlowRule;
-import org.onlab.onos.net.flow.FlowRule.FlowRuleState;
 import org.onlab.onos.net.flow.FlowRuleEvent;
 import org.onlab.onos.net.flow.FlowRuleEvent.Type;
 import org.onlab.onos.net.flow.FlowRuleStore;
@@ -30,18 +30,18 @@ import com.google.common.collect.Multimap;
 /**
  * Manages inventory of flow rules using trivial in-memory implementation.
  */
-//FIXME: I LIE I AM NOT DISTRIBUTED
+//FIXME I LIE. I AIN'T DISTRIBUTED
 @Component(immediate = true)
 @Service
 public class DistributedFlowRuleStore
-extends AbstractStore<FlowRuleEvent, FlowRuleStoreDelegate>
-implements FlowRuleStore {
+        extends AbstractStore<FlowRuleEvent, FlowRuleStoreDelegate>
+        implements FlowRuleStore {
 
     private final Logger log = getLogger(getClass());
 
     // store entries as a pile of rules, no info about device tables
-    private final Multimap<DeviceId, FlowRule> flowEntries =
-            ArrayListMultimap.<DeviceId, FlowRule>create();
+    private final Multimap<DeviceId, FlowEntry> flowEntries =
+            ArrayListMultimap.<DeviceId, FlowEntry>create();
 
     private final Multimap<ApplicationId, FlowRule> flowEntriesById =
             ArrayListMultimap.<ApplicationId, FlowRule>create();
@@ -58,8 +58,13 @@ implements FlowRuleStore {
 
 
     @Override
-    public synchronized FlowRule getFlowRule(FlowRule rule) {
-        for (FlowRule f : flowEntries.get(rule.deviceId())) {
+    public int getFlowRuleCount() {
+        return flowEntries.size();
+    }
+
+    @Override
+    public synchronized FlowEntry getFlowEntry(FlowRule rule) {
+        for (FlowEntry f : flowEntries.get(rule.deviceId())) {
             if (f.equals(rule)) {
                 return f;
             }
@@ -68,8 +73,8 @@ implements FlowRuleStore {
     }
 
     @Override
-    public synchronized Iterable<FlowRule> getFlowEntries(DeviceId deviceId) {
-        Collection<FlowRule> rules = flowEntries.get(deviceId);
+    public synchronized Iterable<FlowEntry> getFlowEntries(DeviceId deviceId) {
+        Collection<FlowEntry> rules = flowEntries.get(deviceId);
         if (rules == null) {
             return Collections.emptyList();
         }
@@ -77,7 +82,7 @@ implements FlowRuleStore {
     }
 
     @Override
-    public synchronized Iterable<FlowRule> getFlowEntriesByAppId(ApplicationId appId) {
+    public synchronized Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId) {
         Collection<FlowRule> rules = flowEntriesById.get(appId);
         if (rules == null) {
             return Collections.emptyList();
@@ -87,7 +92,7 @@ implements FlowRuleStore {
 
     @Override
     public synchronized void storeFlowRule(FlowRule rule) {
-        FlowRule f = new DefaultFlowRule(rule, FlowRuleState.PENDING_ADD);
+        FlowEntry f = new DefaultFlowEntry(rule);
         DeviceId did = f.deviceId();
         if (!flowEntries.containsEntry(did, f)) {
             flowEntries.put(did, f);
@@ -97,57 +102,41 @@ implements FlowRuleStore {
 
     @Override
     public synchronized void deleteFlowRule(FlowRule rule) {
-        FlowRule f = new DefaultFlowRule(rule, FlowRuleState.PENDING_REMOVE);
-        DeviceId did = f.deviceId();
-
-        /*
-         *  find the rule and mark it for deletion.
-         *  Ultimately a flow removed will come remove it.
-         */
-
-        if (flowEntries.containsEntry(did, f)) {
-            //synchronized (flowEntries) {
-            flowEntries.remove(did, f);
-            flowEntries.put(did, f);
-            flowEntriesById.remove(rule.appId(), rule);
-            //}
+        FlowEntry entry = getFlowEntry(rule);
+        if (entry == null) {
+            return;
         }
+        entry.setState(FlowEntryState.PENDING_REMOVE);
     }
 
     @Override
-    public synchronized FlowRuleEvent addOrUpdateFlowRule(FlowRule rule) {
+    public synchronized FlowRuleEvent addOrUpdateFlowRule(FlowEntry rule) {
         DeviceId did = rule.deviceId();
 
         // check if this new rule is an update to an existing entry
-        if (flowEntries.containsEntry(did, rule)) {
-            //synchronized (flowEntries) {
-            // Multimaps support duplicates so we have to remove our rule
-            // and replace it with the current version.
-            flowEntries.remove(did, rule);
-            flowEntries.put(did, rule);
-            //}
+        FlowEntry stored = getFlowEntry(rule);
+        if (stored != null) {
+            stored.setBytes(rule.bytes());
+            stored.setLife(rule.life());
+            stored.setPackets(rule.packets());
+            if (stored.state() == FlowEntryState.PENDING_ADD) {
+                stored.setState(FlowEntryState.ADDED);
+                return new FlowRuleEvent(Type.RULE_ADDED, rule);
+            }
             return new FlowRuleEvent(Type.RULE_UPDATED, rule);
         }
 
         flowEntries.put(did, rule);
-        return new FlowRuleEvent(RULE_ADDED, rule);
+        return null;
     }
 
     @Override
-    public synchronized FlowRuleEvent removeFlowRule(FlowRule rule) {
-        //synchronized (this) {
+    public synchronized FlowRuleEvent removeFlowRule(FlowEntry rule) {
+        // This is where one could mark a rule as removed and still keep it in the store.
         if (flowEntries.remove(rule.deviceId(), rule)) {
             return new FlowRuleEvent(RULE_REMOVED, rule);
         } else {
             return null;
         }
-        //}
     }
-
-
-
-
-
-
-
 }

@@ -42,6 +42,7 @@ import com.google.common.collect.ImmutableSet.Builder;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkState;
 
+//TODO: Add support for multiple provider and annotations
 /**
  * Manages inventory of infrastructure links using a protocol that takes into consideration
  * the order in which events occur.

 /**
  * Implementation of link store using distributed p2p synchronization protocol.
  */
-package org.onlab.onos.store.link.impl;
\ No newline at end of file
+package org.onlab.onos.store.link.impl;

+package org.onlab.onos.store.serializers;
+
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.store.cluster.messaging.ClusterMessage;
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+public final class ClusterMessageSerializer extends Serializer<ClusterMessage> {
+
+    /**
+     * Creates a serializer for {@link ClusterMessage}.
+     */
+    public ClusterMessageSerializer() {
+        // does not accept null
+        super(false);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output, ClusterMessage message) {
+        kryo.writeClassAndObject(output, message.sender());
+        kryo.writeClassAndObject(output, message.subject());
+        output.writeInt(message.payload().length);
+        output.writeBytes(message.payload());
+    }
+
+    @Override
+    public ClusterMessage read(Kryo kryo, Input input,
+                               Class<ClusterMessage> type) {
+        NodeId sender = (NodeId) kryo.readClassAndObject(input);
+        MessageSubject subject = (MessageSubject) kryo.readClassAndObject(input);
+        int payloadSize = input.readInt();
+        byte[] payload = input.readBytes(payloadSize);
+        return new ClusterMessage(sender, subject, payload);
+    }
+}
\ No newline at end of file

 package org.onlab.onos.store.serializers;
 
-import org.onlab.onos.store.impl.OnosTimestamp;
+import org.onlab.onos.store.common.impl.MastershipBasedTimestamp;
 
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.Serializer;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
 
+// To be used if Timestamp ever needs to cross bundle boundary.
 /**
- * Kryo Serializer for {@link OnosTimestamp}.
+ * Kryo Serializer for {@link MastershipBasedTimestamp}.
  */
-public class OnosTimestampSerializer extends Serializer<OnosTimestamp> {
+public class MastershipBasedTimestampSerializer extends Serializer<MastershipBasedTimestamp> {
 
     /**
-     * Default constructor.
+     * Creates a serializer for {@link MastershipBasedTimestamp}.
      */
-    public OnosTimestampSerializer() {
+    public MastershipBasedTimestampSerializer() {
         // non-null, immutable
         super(false, true);
     }
 
     @Override
-    public void write(Kryo kryo, Output output, OnosTimestamp object) {
+    public void write(Kryo kryo, Output output, MastershipBasedTimestamp object) {
         output.writeInt(object.termNumber());
         output.writeInt(object.sequenceNumber());
     }
 
     @Override
-    public OnosTimestamp read(Kryo kryo, Input input, Class<OnosTimestamp> type) {
+    public MastershipBasedTimestamp read(Kryo kryo, Input input, Class<MastershipBasedTimestamp> type) {
         final int term = input.readInt();
         final int sequence = input.readInt();
-        return new OnosTimestamp(term, sequence);
+        return new MastershipBasedTimestamp(term, sequence);
     }
 }

+package org.onlab.onos.store.serializers;
+
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
+
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+public final class MessageSubjectSerializer extends Serializer<MessageSubject> {
+
+    /**
+     * Creates a serializer for {@link MessageSubject}.
+     */
+    public MessageSubjectSerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+
+
+    @Override
+    public void write(Kryo kryo, Output output, MessageSubject object) {
+        output.writeString(object.value());
+    }
+
+    @Override
+    public MessageSubject read(Kryo kryo, Input input,
+            Class<MessageSubject> type) {
+        return new MessageSubject(input.readString());
+    }
+}

@@ -125,7 +125,8 @@ implements TopologyStore {
         // Promote the new topology to current and return a ready-to-send event.
         synchronized (this) {
             current = newTopology;
-            return new TopologyEvent(TopologyEvent.Type.TOPOLOGY_CHANGED, current);
+            return new TopologyEvent(TopologyEvent.Type.TOPOLOGY_CHANGED,
+                                     current, reasons);
         }
     }
 

@@ -7,6 +7,7 @@ import org.junit.Test;
 import org.onlab.onos.cluster.DefaultControllerNode;
 import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.store.cluster.messaging.impl.ClusterCommunicationManager;
+import org.onlab.onos.store.cluster.messaging.impl.MessageSerializer;
 import org.onlab.netty.NettyMessagingService;
 import org.onlab.packet.IpPrefix;
 

+package org.onlab.onos.store.common.impl;
+
+import static org.junit.Assert.*;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+import org.onlab.onos.store.Timestamp;
+import org.onlab.onos.store.serializers.MastershipBasedTimestampSerializer;
+import org.onlab.util.KryoPool;
+
+import com.google.common.testing.EqualsTester;
+
+/**
+ * Test of {@link MastershipBasedTimestamp}.
+ */
+public class MastershipBasedTimestampTest {
+
+    private static final Timestamp TS_1_1 = new MastershipBasedTimestamp(1, 1);
+    private static final Timestamp TS_1_2 = new MastershipBasedTimestamp(1, 2);
+    private static final Timestamp TS_2_1 = new MastershipBasedTimestamp(2, 1);
+    private static final Timestamp TS_2_2 = new MastershipBasedTimestamp(2, 2);
+
+    @Test
+    public final void testBasic() {
+        final int termNumber = 5;
+        final int sequenceNumber = 6;
+        MastershipBasedTimestamp ts = new MastershipBasedTimestamp(termNumber,
+                                                sequenceNumber);
+
+        assertEquals(termNumber, ts.termNumber());
+        assertEquals(sequenceNumber, ts.sequenceNumber());
+    }
+
+    @Test
+    public final void testCompareTo() {
+        assertTrue(TS_1_1.compareTo(TS_1_1) == 0);
+        assertTrue(TS_1_1.compareTo(new MastershipBasedTimestamp(1, 1)) == 0);
+
+        assertTrue(TS_1_1.compareTo(TS_1_2) < 0);
+        assertTrue(TS_1_2.compareTo(TS_1_1) > 0);
+
+        assertTrue(TS_1_2.compareTo(TS_2_1) < 0);
+        assertTrue(TS_1_2.compareTo(TS_2_2) < 0);
+        assertTrue(TS_2_1.compareTo(TS_1_1) > 0);
+        assertTrue(TS_2_2.compareTo(TS_1_1) > 0);
+    }
+
+    @Test
+    public final void testEqualsObject() {
+        new EqualsTester()
+        .addEqualityGroup(new MastershipBasedTimestamp(1, 1),
+                          new MastershipBasedTimestamp(1, 1), TS_1_1)
+        .addEqualityGroup(new MastershipBasedTimestamp(1, 2),
+                          new MastershipBasedTimestamp(1, 2), TS_1_2)
+        .addEqualityGroup(new MastershipBasedTimestamp(2, 1),
+                          new MastershipBasedTimestamp(2, 1), TS_2_1)
+        .addEqualityGroup(new MastershipBasedTimestamp(2, 2),
+                          new MastershipBasedTimestamp(2, 2), TS_2_2)
+        .testEquals();
+    }
+
+    @Test
+    public final void testKryoSerializable() {
+        final ByteBuffer buffer = ByteBuffer.allocate(1 * 1024 * 1024);
+        final KryoPool kryos = KryoPool.newBuilder()
+                .register(MastershipBasedTimestamp.class)
+                .build();
+
+        kryos.serialize(TS_2_1, buffer);
+        buffer.flip();
+        Timestamp copy = kryos.deserialize(buffer);
+
+        new EqualsTester()
+            .addEqualityGroup(TS_2_1, copy)
+            .testEquals();
+    }
+
+    @Test
+    public final void testKryoSerializableWithHandcraftedSerializer() {
+        final ByteBuffer buffer = ByteBuffer.allocate(1 * 1024 * 1024);
+        final KryoPool kryos = KryoPool.newBuilder()
+                .register(MastershipBasedTimestamp.class, new MastershipBasedTimestampSerializer())
+                .build();
+
+        kryos.serialize(TS_1_2, buffer);
+        buffer.flip();
+        Timestamp copy = kryos.deserialize(buffer);
+
+        new EqualsTester()
+            .addEqualityGroup(TS_1_2, copy)
+            .testEquals();
+    }
+
+}

+package org.onlab.onos.store.common.impl;
+
+import static org.junit.Assert.*;
+
+import java.nio.ByteBuffer;
+
+import org.junit.Test;
+import org.onlab.onos.store.Timestamp;
+import org.onlab.util.KryoPool;
+
+import com.google.common.testing.EqualsTester;
+
+/**
+ * Test of {@link Timestamped}.
+ */
+public class TimestampedTest {
+
+    private static final Timestamp TS_1_1 = new MastershipBasedTimestamp(1, 1);
+    private static final Timestamp TS_1_2 = new MastershipBasedTimestamp(1, 2);
+    private static final Timestamp TS_2_1 = new MastershipBasedTimestamp(2, 1);
+
+    @Test
+    public final void testHashCode() {
+        Timestamped<String> a = new Timestamped<>("a", TS_1_1);
+        Timestamped<String> b = new Timestamped<>("b", TS_1_1);
+        assertTrue("value does not impact hashCode",
+                a.hashCode() == b.hashCode());
+    }
+
+    @Test
+    public final void testEquals() {
+        Timestamped<String> a = new Timestamped<>("a", TS_1_1);
+        Timestamped<String> b = new Timestamped<>("b", TS_1_1);
+        assertTrue("value does not impact equality",
+                a.equals(b));
+
+        new EqualsTester()
+        .addEqualityGroup(new Timestamped<>("a", TS_1_1),
+                          new Timestamped<>("b", TS_1_1),
+                          new Timestamped<>("c", TS_1_1))
+        .addEqualityGroup(new Timestamped<>("a", TS_1_2),
+                          new Timestamped<>("b", TS_1_2),
+                          new Timestamped<>("c", TS_1_2))
+        .addEqualityGroup(new Timestamped<>("a", TS_2_1),
+                          new Timestamped<>("b", TS_2_1),
+                          new Timestamped<>("c", TS_2_1))
+        .testEquals();
+
+    }
+
+    @Test
+    public final void testValue() {
+       final Integer n = Integer.valueOf(42);
+       Timestamped<Integer> tsv = new Timestamped<>(n, TS_1_1);
+       assertSame(n, tsv.value());
+
+    }
+
+    @Test(expected = NullPointerException.class)
+    public final void testValueNonNull() {
+        new Timestamped<>(null, TS_1_1);
+    }
+
+    @Test(expected = NullPointerException.class)
+    public final void testTimestampNonNull() {
+        new Timestamped<>("Foo", null);
+    }
+
+    @Test
+    public final void testIsNewer() {
+        Timestamped<String> a = new Timestamped<>("a", TS_1_2);
+        Timestamped<String> b = new Timestamped<>("b", TS_1_1);
+        assertTrue(a.isNewer(b));
+        assertFalse(b.isNewer(a));
+    }
+
+    @Test
+    public final void testKryoSerializable() {
+        final ByteBuffer buffer = ByteBuffer.allocate(1 * 1024 * 1024);
+        final KryoPool kryos = KryoPool.newBuilder()
+                .register(Timestamped.class,
+                        MastershipBasedTimestamp.class)
+                .build();
+
+        Timestamped<String> original = new Timestamped<>("foobar", TS_1_1);
+        kryos.serialize(original, buffer);
+        buffer.flip();
+        Timestamped<String> copy = kryos.deserialize(buffer);
+
+        new EqualsTester()
+            .addEqualityGroup(original, copy)
+            .testEquals();
+    }
+}

+package org.onlab.onos.store.device.impl;
+
+import static org.junit.Assert.*;
+import static org.onlab.onos.net.Device.Type.SWITCH;
+import static org.onlab.onos.net.DeviceId.deviceId;
+import static org.onlab.onos.net.device.DeviceEvent.Type.*;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+import org.onlab.onos.cluster.ClusterEventListener;
+import org.onlab.onos.cluster.ClusterService;
+import org.onlab.onos.cluster.ControllerNode;
+import org.onlab.onos.cluster.ControllerNode.State;
+import org.onlab.onos.cluster.DefaultControllerNode;
+import org.onlab.onos.cluster.MastershipTerm;
+import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.net.Annotations;
+import org.onlab.onos.net.DefaultAnnotations;
+import org.onlab.onos.net.Device;
+import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.Port;
+import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.SparseAnnotations;
+import org.onlab.onos.net.device.DefaultDeviceDescription;
+import org.onlab.onos.net.device.DefaultPortDescription;
+import org.onlab.onos.net.device.DeviceDescription;
+import org.onlab.onos.net.device.DeviceEvent;
+import org.onlab.onos.net.device.DeviceStore;
+import org.onlab.onos.net.device.DeviceStoreDelegate;
+import org.onlab.onos.net.device.PortDescription;
+import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.onos.store.ClockService;
+import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
+import org.onlab.onos.store.cluster.messaging.ClusterMessage;
+import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
+import org.onlab.onos.store.cluster.messaging.MessageSubject;
+import org.onlab.packet.IpPrefix;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+
+// TODO add tests for remote replication
+/**
+ * Test of the gossip based distributed DeviceStore implementation.
+ */
+public class GossipDeviceStoreTest {
+
+    private static final ProviderId PID = new ProviderId("of", "foo");
+    private static final ProviderId PIDA = new ProviderId("of", "bar", true);
+    private static final DeviceId DID1 = deviceId("of:foo");
+    private static final DeviceId DID2 = deviceId("of:bar");
+    private static final String MFR = "whitebox";
+    private static final String HW = "1.1.x";
+    private static final String SW1 = "3.8.1";
+    private static final String SW2 = "3.9.5";
+    private static final String SN = "43311-12345";
+
+    private static final PortNumber P1 = PortNumber.portNumber(1);
+    private static final PortNumber P2 = PortNumber.portNumber(2);
+    private static final PortNumber P3 = PortNumber.portNumber(3);
+
+    private static final SparseAnnotations A1 = DefaultAnnotations.builder()
+            .set("A1", "a1")
+            .set("B1", "b1")
+            .build();
+    private static final SparseAnnotations A1_2 = DefaultAnnotations.builder()
+            .remove("A1")
+            .set("B3", "b3")
+            .build();
+    private static final SparseAnnotations A2 = DefaultAnnotations.builder()
+            .set("A2", "a2")
+            .set("B2", "b2")
+            .build();
+    private static final SparseAnnotations A2_2 = DefaultAnnotations.builder()
+            .remove("A2")
+            .set("B4", "b4")
+            .build();
+
+    private static final NodeId MYSELF = new NodeId("myself");
+
+    private GossipDeviceStore gossipDeviceStore;
+    private DeviceStore deviceStore;
+
+    private DeviceClockManager deviceClockManager;
+    private ClockService clockService;
+
+    @BeforeClass
+    public static void setUpBeforeClass() throws Exception {
+    }
+
+    @AfterClass
+    public static void tearDownAfterClass() throws Exception {
+    }
+
+
+    @Before
+    public void setUp() throws Exception {
+        deviceClockManager = new DeviceClockManager();
+        deviceClockManager.activate();
+        clockService = deviceClockManager;
+
+        deviceClockManager.setMastershipTerm(DID1, MastershipTerm.of(MYSELF, 1));
+        deviceClockManager.setMastershipTerm(DID2, MastershipTerm.of(MYSELF, 2));
+
+        ClusterCommunicationService clusterCommunicator = new TestClusterCommunicationService();
+        ClusterService clusterService = new TestClusterService();
+
+        gossipDeviceStore = new TestGossipDeviceStore(clockService, clusterService, clusterCommunicator);
+        gossipDeviceStore.activate();
+        deviceStore = gossipDeviceStore;
+    }
+
+    @After
+    public void tearDown() throws Exception {
+        gossipDeviceStore.deactivate();
+        deviceClockManager.deactivate();
+    }
+
+    private void putDevice(DeviceId deviceId, String swVersion,
+                           SparseAnnotations... annotations) {
+        DeviceDescription description =
+                new DefaultDeviceDescription(deviceId.uri(), SWITCH, MFR,
+                        HW, swVersion, SN, annotations);
+        deviceStore.createOrUpdateDevice(PID, deviceId, description);
+    }
+
+    private void putDeviceAncillary(DeviceId deviceId, String swVersion,
+                                    SparseAnnotations... annotations) {
+        DeviceDescription description =
+                new DefaultDeviceDescription(deviceId.uri(), SWITCH, MFR,
+                        HW, swVersion, SN, annotations);
+        deviceStore.createOrUpdateDevice(PIDA, deviceId, description);
+    }
+
+    private static void assertDevice(DeviceId id, String swVersion, Device device) {
+        assertNotNull(device);
+        assertEquals(id, device.id());
+        assertEquals(MFR, device.manufacturer());
+        assertEquals(HW, device.hwVersion());
+        assertEquals(swVersion, device.swVersion());
+        assertEquals(SN, device.serialNumber());
+    }
+
+    /**
+     * Verifies that Annotations created by merging {@code annotations} is
+     * equal to actual Annotations.
+     *
+     * @param actual Annotations to check
+     * @param annotations
+     */
+    private static void assertAnnotationsEquals(Annotations actual, SparseAnnotations... annotations) {
+        DefaultAnnotations expected = DefaultAnnotations.builder().build();
+        for (SparseAnnotations a : annotations) {
+            expected = DefaultAnnotations.merge(expected, a);
+        }
+        assertEquals(expected.keys(), actual.keys());
+        for (String key : expected.keys()) {
+            assertEquals(expected.value(key), actual.value(key));
+        }
+    }
+
+    @Test
+    public final void testGetDeviceCount() {
+        assertEquals("initialy empty", 0, deviceStore.getDeviceCount());
+
+        putDevice(DID1, SW1);
+        putDevice(DID2, SW2);
+        putDevice(DID1, SW1);
+
+        assertEquals("expect 2 uniq devices", 2, deviceStore.getDeviceCount());
+    }
+
+    @Test
+    public final void testGetDevices() {
+        assertEquals("initialy empty", 0, Iterables.size(deviceStore.getDevices()));
+
+        putDevice(DID1, SW1);
+        putDevice(DID2, SW2);
+        putDevice(DID1, SW1);
+
+        assertEquals("expect 2 uniq devices",
+                2, Iterables.size(deviceStore.getDevices()));
+
+        Map<DeviceId, Device> devices = new HashMap<>();
+        for (Device device : deviceStore.getDevices()) {
+            devices.put(device.id(), device);
+        }
+
+        assertDevice(DID1, SW1, devices.get(DID1));
+        assertDevice(DID2, SW2, devices.get(DID2));
+
+        // add case for new node?
+    }
+
+    @Test
+    public final void testGetDevice() {
+
+        putDevice(DID1, SW1);
+
+        assertDevice(DID1, SW1, deviceStore.getDevice(DID1));
+        assertNull("DID2 shouldn't be there", deviceStore.getDevice(DID2));
+    }
+
+    @Test
+    public final void testCreateOrUpdateDevice() {
+        DeviceDescription description =
+                new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
+                        HW, SW1, SN);
+        DeviceEvent event = deviceStore.createOrUpdateDevice(PID, DID1, description);
+        assertEquals(DEVICE_ADDED, event.type());
+        assertDevice(DID1, SW1, event.subject());
+
+        DeviceDescription description2 =
+                new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
+                        HW, SW2, SN);
+        DeviceEvent event2 = deviceStore.createOrUpdateDevice(PID, DID1, description2);
+        assertEquals(DEVICE_UPDATED, event2.type());
+        assertDevice(DID1, SW2, event2.subject());
+
+        assertNull("No change expected", deviceStore.createOrUpdateDevice(PID, DID1, description2));
+    }
+
+    @Test
+    public final void testCreateOrUpdateDeviceAncillary() {
+        DeviceDescription description =
+                new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
+                        HW, SW1, SN, A2);
+        DeviceEvent event = deviceStore.createOrUpdateDevice(PIDA, DID1, description);
+        assertEquals(DEVICE_ADDED, event.type());
+        assertDevice(DID1, SW1, event.subject());
+        assertEquals(PIDA, event.subject().providerId());
+        assertAnnotationsEquals(event.subject().annotations(), A2);
+        assertFalse("Ancillary will not bring device up", deviceStore.isAvailable(DID1));
+
+        DeviceDescription description2 =
+                new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
+                        HW, SW2, SN, A1);
+        DeviceEvent event2 = deviceStore.createOrUpdateDevice(PID, DID1, description2);
+        assertEquals(DEVICE_UPDATED, event2.type());
+        assertDevice(DID1, SW2, event2.subject());
+        assertEquals(PID, event2.subject().providerId());
+        assertAnnotationsEquals(event2.subject().annotations(), A1, A2);
+        assertTrue(deviceStore.isAvailable(DID1));
+
+        assertNull("No change expected", deviceStore.createOrUpdateDevice(PID, DID1, description2));
+
+        // For now, Ancillary is ignored once primary appears
+        assertNull("No change expected", deviceStore.createOrUpdateDevice(PIDA, DID1, description));
+
+        // But, Ancillary annotations will be in effect
+        DeviceDescription description3 =
+                new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
+                        HW, SW1, SN, A2_2);
+        DeviceEvent event3 = deviceStore.createOrUpdateDevice(PIDA, DID1, description3);
+        assertEquals(DEVICE_UPDATED, event3.type());
+        // basic information will be the one from Primary
+        assertDevice(DID1, SW2, event3.subject());
+        assertEquals(PID, event3.subject().providerId());
+        // but annotation from Ancillary will be merged
+        assertAnnotationsEquals(event3.subject().annotations(), A1, A2, A2_2);
+        assertTrue(deviceStore.isAvailable(DID1));
+    }
+
+
+    @Test
+    public final void testMarkOffline() {
+
+        putDevice(DID1, SW1);
+        assertTrue(deviceStore.isAvailable(DID1));
+
+        DeviceEvent event = deviceStore.markOffline(DID1);
+        assertEquals(DEVICE_AVAILABILITY_CHANGED, event.type());
+        assertDevice(DID1, SW1, event.subject());
+        assertFalse(deviceStore.isAvailable(DID1));
+
+        DeviceEvent event2 = deviceStore.markOffline(DID1);
+        assertNull("No change, no event", event2);
+}
+
+    @Test
+    public final void testUpdatePorts() {
+        putDevice(DID1, SW1);
+        List<PortDescription> pds = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, true),
+                new DefaultPortDescription(P2, true)
+                );
+
+        List<DeviceEvent> events = deviceStore.updatePorts(PID, DID1, pds);
+
+        Set<PortNumber> expectedPorts = Sets.newHashSet(P1, P2);
+        for (DeviceEvent event : events) {
+            assertEquals(PORT_ADDED, event.type());
+            assertDevice(DID1, SW1, event.subject());
+            assertTrue("PortNumber is one of expected",
+                    expectedPorts.remove(event.port().number()));
+            assertTrue("Port is enabled", event.port().isEnabled());
+        }
+        assertTrue("Event for all expectedport appeared", expectedPorts.isEmpty());
+
+
+        List<PortDescription> pds2 = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, false),
+                new DefaultPortDescription(P2, true),
+                new DefaultPortDescription(P3, true)
+                );
+
+        events = deviceStore.updatePorts(PID, DID1, pds2);
+        assertFalse("event should be triggered", events.isEmpty());
+        for (DeviceEvent event : events) {
+            PortNumber num = event.port().number();
+            if (P1.equals(num)) {
+                assertEquals(PORT_UPDATED, event.type());
+                assertDevice(DID1, SW1, event.subject());
+                assertFalse("Port is disabled", event.port().isEnabled());
+            } else if (P2.equals(num)) {
+                fail("P2 event not expected.");
+            } else if (P3.equals(num)) {
+                assertEquals(PORT_ADDED, event.type());
+                assertDevice(DID1, SW1, event.subject());
+                assertTrue("Port is enabled", event.port().isEnabled());
+            } else {
+                fail("Unknown port number encountered: " + num);
+            }
+        }
+
+        List<PortDescription> pds3 = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, false),
+                new DefaultPortDescription(P2, true)
+                );
+        events = deviceStore.updatePorts(PID, DID1, pds3);
+        assertFalse("event should be triggered", events.isEmpty());
+        for (DeviceEvent event : events) {
+            PortNumber num = event.port().number();
+            if (P1.equals(num)) {
+                fail("P1 event not expected.");
+            } else if (P2.equals(num)) {
+                fail("P2 event not expected.");
+            } else if (P3.equals(num)) {
+                assertEquals(PORT_REMOVED, event.type());
+                assertDevice(DID1, SW1, event.subject());
+                assertTrue("Port was enabled", event.port().isEnabled());
+            } else {
+                fail("Unknown port number encountered: " + num);
+            }
+        }
+
+    }
+
+    @Test
+    public final void testUpdatePortStatus() {
+        putDevice(DID1, SW1);
+        List<PortDescription> pds = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, true)
+                );
+        deviceStore.updatePorts(PID, DID1, pds);
+
+        DeviceEvent event = deviceStore.updatePortStatus(PID, DID1,
+                new DefaultPortDescription(P1, false));
+        assertEquals(PORT_UPDATED, event.type());
+        assertDevice(DID1, SW1, event.subject());
+        assertEquals(P1, event.port().number());
+        assertFalse("Port is disabled", event.port().isEnabled());
+
+    }
+    @Test
+    public final void testUpdatePortStatusAncillary() {
+        putDeviceAncillary(DID1, SW1);
+        putDevice(DID1, SW1);
+        List<PortDescription> pds = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, true, A1)
+                );
+        deviceStore.updatePorts(PID, DID1, pds);
+
+        DeviceEvent event = deviceStore.updatePortStatus(PID, DID1,
+                new DefaultPortDescription(P1, false, A1_2));
+        assertEquals(PORT_UPDATED, event.type());
+        assertDevice(DID1, SW1, event.subject());
+        assertEquals(P1, event.port().number());
+        assertAnnotationsEquals(event.port().annotations(), A1, A1_2);
+        assertFalse("Port is disabled", event.port().isEnabled());
+
+        DeviceEvent event2 = deviceStore.updatePortStatus(PIDA, DID1,
+                new DefaultPortDescription(P1, true));
+        assertNull("Ancillary is ignored if primary exists", event2);
+
+        // but, Ancillary annotation update will be notified
+        DeviceEvent event3 = deviceStore.updatePortStatus(PIDA, DID1,
+                new DefaultPortDescription(P1, true, A2));
+        assertEquals(PORT_UPDATED, event3.type());
+        assertDevice(DID1, SW1, event3.subject());
+        assertEquals(P1, event3.port().number());
+        assertAnnotationsEquals(event3.port().annotations(), A1, A1_2, A2);
+        assertFalse("Port is disabled", event3.port().isEnabled());
+
+        // port only reported from Ancillary will be notified as down
+        DeviceEvent event4 = deviceStore.updatePortStatus(PIDA, DID1,
+                new DefaultPortDescription(P2, true));
+        assertEquals(PORT_ADDED, event4.type());
+        assertDevice(DID1, SW1, event4.subject());
+        assertEquals(P2, event4.port().number());
+        assertAnnotationsEquals(event4.port().annotations());
+        assertFalse("Port is disabled if not given from primary provider",
+                        event4.port().isEnabled());
+    }
+
+    @Test
+    public final void testGetPorts() {
+        putDevice(DID1, SW1);
+        putDevice(DID2, SW1);
+        List<PortDescription> pds = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, true),
+                new DefaultPortDescription(P2, true)
+                );
+        deviceStore.updatePorts(PID, DID1, pds);
+
+        Set<PortNumber> expectedPorts = Sets.newHashSet(P1, P2);
+        List<Port> ports = deviceStore.getPorts(DID1);
+        for (Port port : ports) {
+            assertTrue("Port is enabled", port.isEnabled());
+            assertTrue("PortNumber is one of expected",
+                    expectedPorts.remove(port.number()));
+        }
+        assertTrue("Event for all expectedport appeared", expectedPorts.isEmpty());
+
+
+        assertTrue("DID2 has no ports", deviceStore.getPorts(DID2).isEmpty());
+    }
+
+    @Test
+    public final void testGetPort() {
+        putDevice(DID1, SW1);
+        putDevice(DID2, SW1);
+        List<PortDescription> pds = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, true),
+                new DefaultPortDescription(P2, false)
+                );
+        deviceStore.updatePorts(PID, DID1, pds);
+
+        Port port1 = deviceStore.getPort(DID1, P1);
+        assertEquals(P1, port1.number());
+        assertTrue("Port is enabled", port1.isEnabled());
+
+        Port port2 = deviceStore.getPort(DID1, P2);
+        assertEquals(P2, port2.number());
+        assertFalse("Port is disabled", port2.isEnabled());
+
+        Port port3 = deviceStore.getPort(DID1, P3);
+        assertNull("P3 not expected", port3);
+    }
+
+    @Test
+    public final void testRemoveDevice() {
+        putDevice(DID1, SW1, A1);
+        List<PortDescription> pds = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, true, A2)
+                );
+        deviceStore.updatePorts(PID, DID1, pds);
+        putDevice(DID2, SW1);
+
+        assertEquals(2, deviceStore.getDeviceCount());
+        assertEquals(1, deviceStore.getPorts(DID1).size());
+        assertAnnotationsEquals(deviceStore.getDevice(DID1).annotations(), A1);
+        assertAnnotationsEquals(deviceStore.getPort(DID1, P1).annotations(), A2);
+
+        DeviceEvent event = deviceStore.removeDevice(DID1);
+        assertEquals(DEVICE_REMOVED, event.type());
+        assertDevice(DID1, SW1, event.subject());
+
+        assertEquals(1, deviceStore.getDeviceCount());
+        assertEquals(0, deviceStore.getPorts(DID1).size());
+
+        // putBack Device, Port w/o annotation
+        putDevice(DID1, SW1);
+        List<PortDescription> pds2 = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, true)
+                );
+        deviceStore.updatePorts(PID, DID1, pds2);
+
+        // annotations should not survive
+        assertEquals(2, deviceStore.getDeviceCount());
+        assertEquals(1, deviceStore.getPorts(DID1).size());
+        assertAnnotationsEquals(deviceStore.getDevice(DID1).annotations());
+        assertAnnotationsEquals(deviceStore.getPort(DID1, P1).annotations());
+    }
+
+    // If Delegates should be called only on remote events,
+    // then Simple* should never call them, thus not test required.
+    // TODO add test for Port events when we have them
+    @Ignore("Ignore until Delegate spec. is clear.")
+    @Test
+    public final void testEvents() throws InterruptedException {
+        final CountDownLatch addLatch = new CountDownLatch(1);
+        DeviceStoreDelegate checkAdd = new DeviceStoreDelegate() {
+            @Override
+            public void notify(DeviceEvent event) {
+                assertEquals(DEVICE_ADDED, event.type());
+                assertDevice(DID1, SW1, event.subject());
+                addLatch.countDown();
+            }
+        };
+        final CountDownLatch updateLatch = new CountDownLatch(1);
+        DeviceStoreDelegate checkUpdate = new DeviceStoreDelegate() {
+            @Override
+            public void notify(DeviceEvent event) {
+                assertEquals(DEVICE_UPDATED, event.type());
+                assertDevice(DID1, SW2, event.subject());
+                updateLatch.countDown();
+            }
+        };
+        final CountDownLatch removeLatch = new CountDownLatch(1);
+        DeviceStoreDelegate checkRemove = new DeviceStoreDelegate() {
+            @Override
+            public void notify(DeviceEvent event) {
+                assertEquals(DEVICE_REMOVED, event.type());
+                assertDevice(DID1, SW2, event.subject());
+                removeLatch.countDown();
+            }
+        };
+
+        DeviceDescription description =
+                new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
+                        HW, SW1, SN);
+        deviceStore.setDelegate(checkAdd);
+        deviceStore.createOrUpdateDevice(PID, DID1, description);
+        assertTrue("Add event fired", addLatch.await(1, TimeUnit.SECONDS));
+
+
+        DeviceDescription description2 =
+                new DefaultDeviceDescription(DID1.uri(), SWITCH, MFR,
+                        HW, SW2, SN);
+        deviceStore.unsetDelegate(checkAdd);
+        deviceStore.setDelegate(checkUpdate);
+        deviceStore.createOrUpdateDevice(PID, DID1, description2);
+        assertTrue("Update event fired", updateLatch.await(1, TimeUnit.SECONDS));
+
+        deviceStore.unsetDelegate(checkUpdate);
+        deviceStore.setDelegate(checkRemove);
+        deviceStore.removeDevice(DID1);
+        assertTrue("Remove event fired", removeLatch.await(1, TimeUnit.SECONDS));
+    }
+
+    private static final class TestGossipDeviceStore extends GossipDeviceStore {
+
+        public TestGossipDeviceStore(
+                ClockService clockService,
+                ClusterService clusterService,
+                ClusterCommunicationService clusterCommunicator) {
+            this.clockService = clockService;
+            this.clusterService = clusterService;
+            this.clusterCommunicator = clusterCommunicator;
+        }
+    }
+
+    private static final class TestClusterCommunicationService implements ClusterCommunicationService {
+        @Override
+        public boolean broadcast(ClusterMessage message) throws IOException { return true; }
+        @Override
+        public boolean unicast(ClusterMessage message, NodeId nodeId) throws IOException { return true; }
+        @Override
+        public boolean multicast(ClusterMessage message, Set<NodeId> nodeIds) throws IOException { return true; }
+        @Override
+        public void addSubscriber(MessageSubject subject, ClusterMessageHandler subscriber) {}
+    }
+
+    private static final class TestClusterService implements ClusterService {
+
+        private static final ControllerNode ONOS1 =
+            new DefaultControllerNode(new NodeId("N1"), IpPrefix.valueOf("127.0.0.1"));
+        private final Map<NodeId, ControllerNode> nodes = new HashMap<>();
+        private final Map<NodeId, ControllerNode.State> nodeStates = new HashMap<>();
+
+        public TestClusterService() {
+            nodes.put(new NodeId("N1"), ONOS1);
+            nodeStates.put(new NodeId("N1"), ControllerNode.State.ACTIVE);
+        }
+
+        @Override
+        public ControllerNode getLocalNode() {
+            return ONOS1;
+        }
+
+        @Override
+        public Set<ControllerNode> getNodes() {
+            return Sets.newHashSet(nodes.values());
+        }
+
+        @Override
+        public ControllerNode getNode(NodeId nodeId) {
+            return nodes.get(nodeId);
+        }
+
+        @Override
+        public State getState(NodeId nodeId) {
+            return nodeStates.get(nodeId);
+        }
+
+        @Override
+        public void addListener(ClusterEventListener listener) {
+        }
+
+        @Override
+        public void removeListener(ClusterEventListener listener) {
+        }
+    }
+}

@@ -57,7 +57,7 @@ public class DistributedClusterStore
 
         rawNodes = theInstance.getMap("nodes");
         OptionalCacheLoader<NodeId, DefaultControllerNode> nodeLoader
-                = new OptionalCacheLoader<>(kryoSerializationService, rawNodes);
+                = new OptionalCacheLoader<>(serializer, rawNodes);
         nodes = new AbsentInvalidatingLoadingCache<>(newBuilder().build(nodeLoader));
         rawNodes.addEntryListener(new RemoteCacheEventHandler<>(nodes), true);
 

@@ -30,8 +30,7 @@ import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.store.common.StoreManager;
 import org.onlab.onos.store.common.StoreService;
 import org.onlab.onos.store.common.TestStoreManager;
-import org.onlab.onos.store.serializers.KryoSerializationManager;
-import org.onlab.onos.store.serializers.KryoSerializationService;
+import org.onlab.onos.store.serializers.KryoSerializer;
 import org.onlab.packet.IpPrefix;
 
 import com.google.common.collect.Sets;
@@ -57,7 +56,7 @@ public class DistributedMastershipStoreTest {
 
     private DistributedMastershipStore dms;
     private TestDistributedMastershipStore testStore;
-    private KryoSerializationManager serializationMgr;
+    private KryoSerializer serializationMgr;
     private StoreManager storeMgr;
 
     @BeforeClass
@@ -76,8 +75,7 @@ public class DistributedMastershipStoreTest {
         storeMgr = new TestStoreManager(Hazelcast.newHazelcastInstance(config));
         storeMgr.activate();
 
-        serializationMgr = new KryoSerializationManager();
-        serializationMgr.activate();
+        serializationMgr = new KryoSerializer();
 
         dms = new TestDistributedMastershipStore(storeMgr, serializationMgr);
         dms.clusterService = new TestClusterService();
@@ -90,8 +88,6 @@ public class DistributedMastershipStoreTest {
     public void tearDown() throws Exception {
         dms.deactivate();
 
-        serializationMgr.deactivate();
-
         storeMgr.deactivate();
     }
 
@@ -234,9 +230,9 @@ public class DistributedMastershipStoreTest {
     private class TestDistributedMastershipStore extends
             DistributedMastershipStore {
         public TestDistributedMastershipStore(StoreService storeService,
-                KryoSerializationService kryoSerializationService) {
+                KryoSerializer kryoSerialization) {
             this.storeService = storeService;
-            this.kryoSerializationService = kryoSerializationService;
+            this.serializer = kryoSerialization;
         }
 
         //helper to populate master/backup structures
@@ -260,6 +256,7 @@ public class DistributedMastershipStoreTest {
             }
         }
 
+        //a dumb utility function.
         public void dump() {
             System.out.println("standbys");
             for (Map.Entry<byte [], byte []> e : standbys.entrySet()) {

@@ -15,7 +15,8 @@ import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.onos.event.Event;
 import org.onlab.onos.store.AbstractStore;
 import org.onlab.onos.store.StoreDelegate;
-import org.onlab.onos.store.serializers.KryoSerializationService;
+import org.onlab.onos.store.serializers.KryoSerializer;
+import org.onlab.onos.store.serializers.StoreSerializer;
 import org.slf4j.Logger;
 
 import static com.google.common.base.Preconditions.checkNotNull;
@@ -24,7 +25,7 @@ import static org.slf4j.LoggerFactory.getLogger;
 /**
  * Abstraction of a distributed store based on Hazelcast.
  */
-@Component(componentAbstract = true)
+@Component
 public abstract class AbstractHazelcastStore<E extends Event, D extends StoreDelegate<E>>
         extends AbstractStore<E, D> {
 
@@ -33,13 +34,13 @@ public abstract class AbstractHazelcastStore<E extends Event, D extends StoreDel
     @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
     protected StoreService storeService;
 
-    @Reference(cardinality = ReferenceCardinality.MANDATORY_UNARY)
-    protected KryoSerializationService kryoSerializationService;
+    protected StoreSerializer serializer;
 
     protected HazelcastInstance theInstance;
 
     @Activate
     public void activate() {
+        serializer = new KryoSerializer();
         theInstance = storeService.getHazelcastInstance();
     }
 
@@ -50,7 +51,7 @@ public abstract class AbstractHazelcastStore<E extends Event, D extends StoreDel
      * @return serialized object
      */
     protected byte[] serialize(Object obj) {
-        return kryoSerializationService.serialize(obj);
+        return serializer.encode(obj);
     }
 
     /**
@@ -61,7 +62,7 @@ public abstract class AbstractHazelcastStore<E extends Event, D extends StoreDel
      * @return deserialized object
      */
     protected <T> T deserialize(byte[] bytes) {
-        return kryoSerializationService.deserialize(bytes);
+        return serializer.decode(bytes);
     }
 
 

@@ -2,7 +2,7 @@ package org.onlab.onos.store.common;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import org.onlab.onos.store.serializers.KryoSerializationService;
+import org.onlab.onos.store.serializers.StoreSerializer;
 
 import com.google.common.base.Optional;
 import com.google.common.cache.CacheLoader;
@@ -18,28 +18,28 @@ import com.hazelcast.core.IMap;
 public final class OptionalCacheLoader<K, V> extends
         CacheLoader<K, Optional<V>> {
 
-    private final KryoSerializationService kryoSerializationService;
+    private final StoreSerializer serializer;
     private IMap<byte[], byte[]> rawMap;
 
     /**
      * Constructor.
      *
-     * @param kryoSerializationService to use for serialization
+     * @param serializer to use for serialization
      * @param rawMap underlying IMap
      */
-    public OptionalCacheLoader(KryoSerializationService kryoSerializationService, IMap<byte[], byte[]> rawMap) {
-        this.kryoSerializationService = checkNotNull(kryoSerializationService);
+    public OptionalCacheLoader(StoreSerializer serializer, IMap<byte[], byte[]> rawMap) {
+        this.serializer = checkNotNull(serializer);
         this.rawMap = checkNotNull(rawMap);
     }
 
     @Override
     public Optional<V> load(K key) throws Exception {
-        byte[] keyBytes = kryoSerializationService.serialize(key);
+        byte[] keyBytes = serializer.encode(key);
         byte[] valBytes = rawMap.get(keyBytes);
         if (valBytes == null) {
             return Optional.absent();
         }
-        V dev = kryoSerializationService.deserialize(valBytes);
+        V dev = serializer.decode(valBytes);
         return Optional.of(dev);
     }
 }

@@ -47,6 +47,7 @@ import static com.google.common.cache.CacheBuilder.newBuilder;
 import static org.onlab.onos.net.device.DeviceEvent.Type.*;
 import static org.slf4j.LoggerFactory.getLogger;
 
+//TODO: Add support for multiple provider and annotations
 /**
  * Manages inventory of infrastructure devices using Hazelcast-backed map.
  */
@@ -87,7 +88,7 @@ public class DistributedDeviceStore
         // TODO decide on Map name scheme to avoid collision
         rawDevices = theInstance.getMap("devices");
         final OptionalCacheLoader<DeviceId, DefaultDevice> deviceLoader
-                = new OptionalCacheLoader<>(kryoSerializationService, rawDevices);
+                = new OptionalCacheLoader<>(serializer, rawDevices);
         devices = new AbsentInvalidatingLoadingCache<>(newBuilder().build(deviceLoader));
         // refresh/populate cache based on notification from other instance
         devicesListener = rawDevices.addEntryListener(new RemoteDeviceEventHandler(devices), includeValue);
@@ -97,7 +98,7 @@ public class DistributedDeviceStore
 
         rawDevicePorts = theInstance.getMap("devicePorts");
         final OptionalCacheLoader<DeviceId, Map<PortNumber, Port>> devicePortLoader
-                = new OptionalCacheLoader<>(kryoSerializationService, rawDevicePorts);
+                = new OptionalCacheLoader<>(serializer, rawDevicePorts);
         devicePorts = new AbsentInvalidatingLoadingCache<>(newBuilder().build(devicePortLoader));
         // refresh/populate cache based on notification from other instance
         portsListener = rawDevicePorts.addEntryListener(new RemotePortEventHandler(devicePorts), includeValue);

@@ -4,27 +4,15 @@ import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.onos.cluster.MastershipTerm;
 import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.store.ClockService;
-import org.onlab.onos.store.Timestamp;
+import org.onlab.onos.store.ClockProviderService;
 
 // FIXME: Code clone in onos-core-trivial, onos-core-hz-net
 /**
- * Dummy implementation of {@link ClockService}.
+ * Dummy implementation of {@link ClockProviderService}.
  */
 @Component(immediate = true)
 @Service
-public class NoOpClockService implements ClockService {
-
-    @Override
-    public Timestamp getTimestamp(DeviceId deviceId) {
-        return new Timestamp() {
-
-            @Override
-            public int compareTo(Timestamp o) {
-                throw new IllegalStateException("Never expected to be used.");
-            }
-        };
-    }
+public class NoOpClockProviderService implements ClockProviderService {
 
     @Override
     public void setMastershipTerm(DeviceId deviceId, MastershipTerm term) {

 package org.onlab.onos.store.flow.impl;
 
-import static org.onlab.onos.net.flow.FlowRuleEvent.Type.RULE_ADDED;
 import static org.onlab.onos.net.flow.FlowRuleEvent.Type.RULE_REMOVED;
 import static org.slf4j.LoggerFactory.getLogger;
 
@@ -13,9 +12,10 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.net.flow.DefaultFlowRule;
+import org.onlab.onos.net.flow.DefaultFlowEntry;
+import org.onlab.onos.net.flow.FlowEntry;
+import org.onlab.onos.net.flow.FlowEntry.FlowEntryState;
 import org.onlab.onos.net.flow.FlowRule;
-import org.onlab.onos.net.flow.FlowRule.FlowRuleState;
 import org.onlab.onos.net.flow.FlowRuleEvent;
 import org.onlab.onos.net.flow.FlowRuleEvent.Type;
 import org.onlab.onos.net.flow.FlowRuleStore;
@@ -28,20 +28,20 @@ import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Multimap;
 
 /**
- * TEMPORARY: Manages inventory of flow rules using distributed store implementation.
+ * Manages inventory of flow rules using trivial in-memory implementation.
  */
-//FIXME: I LIE I AM NOT DISTRIBUTED
+//FIXME I LIE. I AIN'T DISTRIBUTED
 @Component(immediate = true)
 @Service
 public class DistributedFlowRuleStore
-extends AbstractStore<FlowRuleEvent, FlowRuleStoreDelegate>
-implements FlowRuleStore {
+        extends AbstractStore<FlowRuleEvent, FlowRuleStoreDelegate>
+        implements FlowRuleStore {
 
     private final Logger log = getLogger(getClass());
 
     // store entries as a pile of rules, no info about device tables
-    private final Multimap<DeviceId, FlowRule> flowEntries =
-            ArrayListMultimap.<DeviceId, FlowRule>create();
+    private final Multimap<DeviceId, FlowEntry> flowEntries =
+            ArrayListMultimap.<DeviceId, FlowEntry>create();
 
     private final Multimap<ApplicationId, FlowRule> flowEntriesById =
             ArrayListMultimap.<ApplicationId, FlowRule>create();
@@ -58,8 +58,13 @@ implements FlowRuleStore {
 
 
     @Override
-    public synchronized FlowRule getFlowRule(FlowRule rule) {
-        for (FlowRule f : flowEntries.get(rule.deviceId())) {
+    public int getFlowRuleCount() {
+        return flowEntries.size();
+    }
+
+    @Override
+    public synchronized FlowEntry getFlowEntry(FlowRule rule) {
+        for (FlowEntry f : flowEntries.get(rule.deviceId())) {
             if (f.equals(rule)) {
                 return f;
             }
@@ -68,8 +73,8 @@ implements FlowRuleStore {
     }
 
     @Override
-    public synchronized Iterable<FlowRule> getFlowEntries(DeviceId deviceId) {
-        Collection<FlowRule> rules = flowEntries.get(deviceId);
+    public synchronized Iterable<FlowEntry> getFlowEntries(DeviceId deviceId) {
+        Collection<FlowEntry> rules = flowEntries.get(deviceId);
         if (rules == null) {
             return Collections.emptyList();
         }
@@ -77,7 +82,7 @@ implements FlowRuleStore {
     }
 
     @Override
-    public synchronized Iterable<FlowRule> getFlowEntriesByAppId(ApplicationId appId) {
+    public synchronized Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId) {
         Collection<FlowRule> rules = flowEntriesById.get(appId);
         if (rules == null) {
             return Collections.emptyList();
@@ -87,7 +92,7 @@ implements FlowRuleStore {
 
     @Override
     public synchronized void storeFlowRule(FlowRule rule) {
-        FlowRule f = new DefaultFlowRule(rule, FlowRuleState.PENDING_ADD);
+        FlowEntry f = new DefaultFlowEntry(rule);
         DeviceId did = f.deviceId();
         if (!flowEntries.containsEntry(did, f)) {
             flowEntries.put(did, f);
@@ -97,57 +102,41 @@ implements FlowRuleStore {
 
     @Override
     public synchronized void deleteFlowRule(FlowRule rule) {
-        FlowRule f = new DefaultFlowRule(rule, FlowRuleState.PENDING_REMOVE);
-        DeviceId did = f.deviceId();
-
-        /*
-         *  find the rule and mark it for deletion.
-         *  Ultimately a flow removed will come remove it.
-         */
-
-        if (flowEntries.containsEntry(did, f)) {
-            //synchronized (flowEntries) {
-            flowEntries.remove(did, f);
-            flowEntries.put(did, f);
-            flowEntriesById.remove(rule.appId(), rule);
-            //}
+        FlowEntry entry = getFlowEntry(rule);
+        if (entry == null) {
+            return;
         }
+        entry.setState(FlowEntryState.PENDING_REMOVE);
     }
 
     @Override
-    public synchronized FlowRuleEvent addOrUpdateFlowRule(FlowRule rule) {
+    public synchronized FlowRuleEvent addOrUpdateFlowRule(FlowEntry rule) {
         DeviceId did = rule.deviceId();
 
         // check if this new rule is an update to an existing entry
-        if (flowEntries.containsEntry(did, rule)) {
-            //synchronized (flowEntries) {
-            // Multimaps support duplicates so we have to remove our rule
-            // and replace it with the current version.
-            flowEntries.remove(did, rule);
-            flowEntries.put(did, rule);
-            //}
+        FlowEntry stored = getFlowEntry(rule);
+        if (stored != null) {
+            stored.setBytes(rule.bytes());
+            stored.setLife(rule.life());
+            stored.setPackets(rule.packets());
+            if (stored.state() == FlowEntryState.PENDING_ADD) {
+                stored.setState(FlowEntryState.ADDED);
+                return new FlowRuleEvent(Type.RULE_ADDED, rule);
+            }
             return new FlowRuleEvent(Type.RULE_UPDATED, rule);
         }
 
         flowEntries.put(did, rule);
-        return new FlowRuleEvent(RULE_ADDED, rule);
+        return null;
     }
 
     @Override
-    public synchronized FlowRuleEvent removeFlowRule(FlowRule rule) {
-        //synchronized (this) {
+    public synchronized FlowRuleEvent removeFlowRule(FlowEntry rule) {
+        // This is where one could mark a rule as removed and still keep it in the store.
         if (flowEntries.remove(rule.deviceId(), rule)) {
             return new FlowRuleEvent(RULE_REMOVED, rule);
         } else {
             return null;
         }
-        //}
     }
-
-
-
-
-
-
-
 }

@@ -38,6 +38,7 @@ import com.google.common.collect.Multimap;
 import com.google.common.collect.ImmutableSet.Builder;
 import com.hazelcast.core.IMap;
 
+//TODO: Add support for multiple provider and annotations
 /**
  * Manages inventory of infrastructure links using Hazelcast-backed map.
  */
@@ -70,7 +71,7 @@ public class DistributedLinkStore
         // TODO decide on Map name scheme to avoid collision
         rawLinks = theInstance.getMap("links");
         final OptionalCacheLoader<LinkKey, DefaultLink> linkLoader
-                = new OptionalCacheLoader<>(kryoSerializationService, rawLinks);
+                = new OptionalCacheLoader<>(serializer, rawLinks);
         links = new AbsentInvalidatingLoadingCache<>(newBuilder().build(linkLoader));
         // refresh/populate cache based on notification from other instance
         linksListener = rawLinks.addEntryListener(new RemoteLinkEventHandler(links), includeValue);

@@ -125,7 +125,8 @@ implements TopologyStore {
         // Promote the new topology to current and return a ready-to-send event.
         synchronized (this) {
             current = newTopology;
-            return new TopologyEvent(TopologyEvent.Type.TOPOLOGY_CHANGED, current);
+            return new TopologyEvent(TopologyEvent.Type.TOPOLOGY_CHANGED,
+                                     current, reasons);
         }
     }
 

@@ -36,9 +36,6 @@ import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.common.StoreManager;
 import org.onlab.onos.store.common.StoreService;
 import org.onlab.onos.store.common.TestStoreManager;
-import org.onlab.onos.store.serializers.KryoSerializationManager;
-import org.onlab.onos.store.serializers.KryoSerializationService;
-
 import com.google.common.collect.Iterables;
 import com.google.common.collect.Sets;
 import com.hazelcast.config.Config;
@@ -63,7 +60,6 @@ public class DistributedDeviceStoreTest {
     private static final PortNumber P3 = PortNumber.portNumber(3);
 
     private DistributedDeviceStore deviceStore;
-    private KryoSerializationManager serializationMgr;
 
     private StoreManager storeManager;
 
@@ -85,10 +81,7 @@ public class DistributedDeviceStoreTest {
         storeManager = new TestStoreManager(Hazelcast.newHazelcastInstance(config));
         storeManager.activate();
 
-        serializationMgr = new KryoSerializationManager();
-        serializationMgr.activate();
-
-        deviceStore = new TestDistributedDeviceStore(storeManager, serializationMgr);
+        deviceStore = new TestDistributedDeviceStore(storeManager);
         deviceStore.activate();
     }
 
@@ -96,8 +89,6 @@ public class DistributedDeviceStoreTest {
     public void tearDown() throws Exception {
         deviceStore.deactivate();
 
-        serializationMgr.deactivate();
-
         storeManager.deactivate();
     }
 
@@ -392,10 +383,8 @@ public class DistributedDeviceStoreTest {
     }
 
     private class TestDistributedDeviceStore extends DistributedDeviceStore {
-        public TestDistributedDeviceStore(StoreService storeService,
-                                KryoSerializationService kryoSerializationService) {
+        public TestDistributedDeviceStore(StoreService storeService) {
             this.storeService = storeService;
-            this.kryoSerializationService = kryoSerializationService;
         }
     }
 }

@@ -30,9 +30,6 @@ import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.common.StoreManager;
 import org.onlab.onos.store.common.StoreService;
 import org.onlab.onos.store.common.TestStoreManager;
-import org.onlab.onos.store.serializers.KryoSerializationManager;
-import org.onlab.onos.store.serializers.KryoSerializationService;
-
 import com.google.common.collect.Iterables;
 import com.hazelcast.config.Config;
 import com.hazelcast.core.Hazelcast;
@@ -51,7 +48,6 @@ public class DistributedLinkStoreTest {
     private static final PortNumber P3 = PortNumber.portNumber(3);
 
     private StoreManager storeManager;
-    private KryoSerializationManager serializationMgr;
 
     private DistributedLinkStore linkStore;
 
@@ -71,17 +67,13 @@ public class DistributedLinkStoreTest {
         storeManager = new TestStoreManager(Hazelcast.newHazelcastInstance(config));
         storeManager.activate();
 
-        serializationMgr = new KryoSerializationManager();
-        serializationMgr.activate();
-
-        linkStore = new TestDistributedLinkStore(storeManager, serializationMgr);
+        linkStore = new TestDistributedLinkStore(storeManager);
         linkStore.activate();
     }
 
     @After
     public void tearDown() throws Exception {
         linkStore.deactivate();
-        serializationMgr.deactivate();
         storeManager.deactivate();
     }
 
@@ -361,10 +353,8 @@ public class DistributedLinkStoreTest {
 
 
     class TestDistributedLinkStore extends DistributedLinkStore {
-        TestDistributedLinkStore(StoreService storeService,
-                            KryoSerializationService kryoSerializationService) {
+        TestDistributedLinkStore(StoreService storeService) {
             this.storeService = storeService;
-            this.kryoSerializationService = kryoSerializationService;
         }
     }
 }

@@ -3,7 +3,6 @@ package org.onlab.onos.store.serializers;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.ElementId;
 import org.onlab.onos.net.PortNumber;
-
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.Serializer;
 import com.esotericsoftware.kryo.io.Input;
@@ -15,7 +14,7 @@ import com.esotericsoftware.kryo.io.Output;
 public class ConnectPointSerializer extends Serializer<ConnectPoint> {
 
     /**
-     * Default constructor.
+     * Creates {@link ConnectPointSerializer} serializer instance.
      */
     public ConnectPointSerializer() {
         // non-null, immutable

@@ -16,7 +16,7 @@ import com.esotericsoftware.kryo.io.Output;
 public class DefaultLinkSerializer extends Serializer<DefaultLink> {
 
     /**
-     * Default constructor.
+     * Creates {@link DefaultLink} serializer instance.
      */
     public DefaultLinkSerializer() {
         // non-null, immutable

@@ -16,7 +16,7 @@ public final class DefaultPortSerializer extends
         Serializer<DefaultPort> {
 
     /**
-     * Default constructor.
+     * Creates {@link DefaultPort} serializer instance.
      */
     public DefaultPortSerializer() {
         // non-null, immutable

@@ -14,6 +14,14 @@ import com.esotericsoftware.kryo.io.Output;
 */
 public final class DeviceIdSerializer extends Serializer<DeviceId> {
 
+    /**
+     * Creates {@link DeviceId} serializer instance.
+     */
+    public DeviceIdSerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+
     @Override
     public void write(Kryo kryo, Output output, DeviceId object) {
         kryo.writeObject(output, object.uri());

@@ -19,6 +19,9 @@ public class ImmutableMapSerializer extends FamilySerializer<ImmutableMap<?, ?>>
 
     private final MapSerializer mapSerializer = new MapSerializer();
 
+    /**
+     * Creates {@link ImmutableMap} serializer instance.
+     */
     public ImmutableMapSerializer() {
         // non-null, immutable
         super(false, true);

@@ -18,6 +18,9 @@ public class ImmutableSetSerializer extends FamilySerializer<ImmutableSet<?>> {
 
     private final CollectionSerializer serializer = new CollectionSerializer();
 
+    /**
+     * Creates {@link ImmutableSet} serializer instance.
+     */
     public ImmutableSetSerializer() {
         // non-null, immutable
         super(false, true);

+package org.onlab.onos.store.serializers;
+
+import org.onlab.packet.IpAddress;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+/**
+ * Kryo Serializer for {@link IpAddress}.
+ */
+public class IpAddressSerializer extends Serializer<IpAddress> {
+
+    /**
+     * Creates {@link IpAddress} serializer instance.
+     */
+    public IpAddressSerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+
+    @Override
+    public void write(Kryo kryo, Output output,
+            IpAddress object) {
+        byte[] octs = object.toOctets();
+        output.writeInt(octs.length);
+        output.writeBytes(octs);
+        output.writeInt(object.prefixLength());
+    }
+
+    @Override
+    public IpAddress read(Kryo kryo, Input input,
+            Class<IpAddress> type) {
+        int octLen = input.readInt();
+        byte[] octs = new byte[octLen];
+        input.read(octs);
+        int prefLen = input.readInt();
+        return IpAddress.valueOf(octs, prefLen);
+    }
+
+}

@@ -13,7 +13,7 @@ import com.esotericsoftware.kryo.io.Output;
 public final class IpPrefixSerializer extends Serializer<IpPrefix> {
 
     /**
-     * Default constructor.
+     * Creates {@link IpPrefix} serializer instance.
      */
     public IpPrefixSerializer() {
         // non-null, immutable

 package org.onlab.onos.store.serializers;
 
-import de.javakaffee.kryoserializers.URISerializer;
-import org.apache.felix.scr.annotations.Activate;
-import org.apache.felix.scr.annotations.Component;
-import org.apache.felix.scr.annotations.Deactivate;
-import org.apache.felix.scr.annotations.Service;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+
 import org.onlab.onos.cluster.ControllerNode;
 import org.onlab.onos.cluster.DefaultControllerNode;
+import org.onlab.onos.cluster.MastershipTerm;
 import org.onlab.onos.cluster.NodeId;
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.DefaultAnnotations;
@@ -21,95 +22,64 @@ import org.onlab.onos.net.LinkKey;
 import org.onlab.onos.net.MastershipRole;
 import org.onlab.onos.net.Port;
 import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.device.DefaultDeviceDescription;
+import org.onlab.onos.net.device.DefaultPortDescription;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpPrefix;
 import org.onlab.util.KryoPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.net.URI;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
-
-/**
- * Serialization service using Kryo.
- */
-@Component(immediate = true)
-@Service
-public class KryoSerializationManager implements KryoSerializationService {
-
-    private final Logger log = LoggerFactory.getLogger(getClass());
-    private KryoPool serializerPool;
 
+import de.javakaffee.kryoserializers.URISerializer;
 
-    @Activate
-    public void activate() {
-        setupKryoPool();
-        log.info("Started");
-    }
-
-    @Deactivate
-    public void deactivate() {
-        log.info("Stopped");
-    }
+public final class KryoPoolUtil {
 
     /**
-     * Sets up the common serialzers pool.
+     * KryoPool which can serialize ON.lab misc classes.
      */
-    protected void setupKryoPool() {
-        // FIXME Slice out types used in common to separate pool/namespace.
-        serializerPool = KryoPool.newBuilder()
-                .register(ArrayList.class,
-                          HashMap.class,
-
-                          ControllerNode.State.class,
-                          Device.Type.class,
-
-                          DefaultAnnotations.class,
-                          DefaultControllerNode.class,
-                          DefaultDevice.class,
-                          MastershipRole.class,
-                          Port.class,
-                          Element.class,
-
-                          Link.Type.class
-                )
-                .register(IpPrefix.class, new IpPrefixSerializer())
-                .register(URI.class, new URISerializer())
-                .register(NodeId.class, new NodeIdSerializer())
-                .register(ProviderId.class, new ProviderIdSerializer())
-                .register(DeviceId.class, new DeviceIdSerializer())
-                .register(PortNumber.class, new PortNumberSerializer())
-                .register(DefaultPort.class, new DefaultPortSerializer())
-                .register(LinkKey.class, new LinkKeySerializer())
-                .register(ConnectPoint.class, new ConnectPointSerializer())
-                .register(DefaultLink.class, new DefaultLinkSerializer())
-                .build()
-                .populate(1);
-    }
-
-    @Override
-    public byte[] serialize(final Object obj) {
-        return serializerPool.serialize(obj);
-    }
-
-    @Override
-    public <T> T deserialize(final byte[] bytes) {
-        if (bytes == null) {
-            return null;
-        }
-        return serializerPool.deserialize(bytes);
-    }
-
-    @Override
-    public void serialize(Object obj, ByteBuffer buffer) {
-        serializerPool.serialize(obj, buffer);
-    }
-
-    @Override
-    public <T> T deserialize(ByteBuffer buffer) {
-        return serializerPool.deserialize(buffer);
-    }
+    public static final KryoPool MISC = KryoPool.newBuilder()
+            .register(IpPrefix.class, new IpPrefixSerializer())
+            .register(IpAddress.class, new IpAddressSerializer())
+            .build();
 
+    // TODO: Populate other classes
+    /**
+     * KryoPool which can serialize API bundle classes.
+     */
+    public static final KryoPool API = KryoPool.newBuilder()
+            .register(MISC)
+            .register(
+                    //
+                    ArrayList.class,
+                    Arrays.asList().getClass(),
+                    HashMap.class,
+                    //
+                    ControllerNode.State.class,
+                    Device.Type.class,
+                    DefaultAnnotations.class,
+                    DefaultControllerNode.class,
+                    DefaultDevice.class,
+                    DefaultDeviceDescription.class,
+                    MastershipRole.class,
+                    Port.class,
+                    DefaultPortDescription.class,
+                    Element.class,
+                    Link.Type.class
+                    )
+            .register(URI.class, new URISerializer())
+            .register(NodeId.class, new NodeIdSerializer())
+            .register(ProviderId.class, new ProviderIdSerializer())
+            .register(DeviceId.class, new DeviceIdSerializer())
+            .register(PortNumber.class, new PortNumberSerializer())
+            .register(DefaultPort.class, new DefaultPortSerializer())
+            .register(LinkKey.class, new LinkKeySerializer())
+            .register(ConnectPoint.class, new ConnectPointSerializer())
+            .register(DefaultLink.class, new DefaultLinkSerializer())
+            .register(MastershipTerm.class, new MastershipTermSerializer())
+            .register(MastershipRole.class, new MastershipRoleSerializer())
+
+            .build();
+
+
+    // not to be instantiated
+    private KryoPoolUtil() {}
 }

+package org.onlab.onos.store.serializers;
+
+import org.onlab.util.KryoPool;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+
+/**
+ * StoreSerializer implementation using Kryo.
+ */
+public class KryoSerializer implements StoreSerializer {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
+    protected KryoPool serializerPool;
+
+
+    public KryoSerializer() {
+        setupKryoPool();
+    }
+
+    /**
+     * Sets up the common serialzers pool.
+     */
+    protected void setupKryoPool() {
+        serializerPool = KryoPool.newBuilder()
+                .register(KryoPoolUtil.API)
+                .build()
+                .populate(1);
+    }
+
+    @Override
+    public byte[] encode(final Object obj) {
+        return serializerPool.serialize(obj);
+    }
+
+    @Override
+    public <T> T decode(final byte[] bytes) {
+        if (bytes == null) {
+            return null;
+        }
+        return serializerPool.deserialize(bytes);
+    }
+
+    @Override
+    public void encode(Object obj, ByteBuffer buffer) {
+        serializerPool.serialize(obj, buffer);
+    }
+
+    @Override
+    public <T> T decode(ByteBuffer buffer) {
+        return serializerPool.deserialize(buffer);
+    }
+
+}

@@ -2,6 +2,7 @@ package org.onlab.onos.store.serializers;
 
 import org.onlab.onos.net.ConnectPoint;
 import org.onlab.onos.net.LinkKey;
+
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.Serializer;
 import com.esotericsoftware.kryo.io.Input;
@@ -13,7 +14,7 @@ import com.esotericsoftware.kryo.io.Output;
 public class LinkKeySerializer extends Serializer<LinkKey> {
 
     /**
-     * Default constructor.
+     * Creates {@link LinkKey} serializer instance.
      */
     public LinkKeySerializer() {
         // non-null, immutable

@@ -12,6 +12,14 @@ import com.esotericsoftware.kryo.io.Output;
  */
 public class MastershipRoleSerializer extends Serializer<MastershipRole> {
 
+    /**
+     * Creates {@link MastershipRole} serializer instance.
+     */
+    public MastershipRoleSerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+
     @Override
     public MastershipRole read(Kryo kryo, Input input, Class<MastershipRole> type) {
         final String role = kryo.readObject(input, String.class);

@@ -2,7 +2,6 @@ package org.onlab.onos.store.serializers;
 
 import org.onlab.onos.cluster.MastershipTerm;
 import org.onlab.onos.cluster.NodeId;
-
 import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.Serializer;
 import com.esotericsoftware.kryo.io.Input;
@@ -13,9 +12,17 @@ import com.esotericsoftware.kryo.io.Output;
  */
 public class MastershipTermSerializer extends Serializer<MastershipTerm> {
 
+    /**
+     * Creates {@link MastershipTerm} serializer instance.
+     */
+    public MastershipTermSerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+
     @Override
     public MastershipTerm read(Kryo kryo, Input input, Class<MastershipTerm> type) {
-        final NodeId node = new NodeId(kryo.readObject(input, String.class));
+        final NodeId node = new NodeId(input.readString());
         final int term = input.readInt();
         return MastershipTerm.of(node, term);
     }

@@ -4,6 +4,7 @@ import com.esotericsoftware.kryo.Kryo;
 import com.esotericsoftware.kryo.Serializer;
 import com.esotericsoftware.kryo.io.Input;
 import com.esotericsoftware.kryo.io.Output;
+
 import org.onlab.onos.cluster.NodeId;
 
 /**
@@ -11,14 +12,22 @@ import org.onlab.onos.cluster.NodeId;
  */
 public final class NodeIdSerializer extends Serializer<NodeId> {
 
+    /**
+     * Creates {@link NodeId} serializer instance.
+     */
+    public NodeIdSerializer() {
+        // non-null, immutable
+        super(false, true);
+    }
+
     @Override
     public void write(Kryo kryo, Output output, NodeId object) {
-        kryo.writeObject(output, object.toString());
+        output.writeString(object.toString());
     }
 
     @Override
     public NodeId read(Kryo kryo, Input input, Class<NodeId> type) {
-        final String id = kryo.readObject(input, String.class);
+        final String id = input.readString();
         return new NodeId(id);
     }
 }

@@ -14,7 +14,7 @@ public final class PortNumberSerializer extends
         Serializer<PortNumber> {
 
     /**
-     * Default constructor.
+     * Creates {@link PortNumber} serializer instance.
      */
     public PortNumberSerializer() {
         // non-null, immutable

@@ -13,7 +13,7 @@ import com.esotericsoftware.kryo.io.Output;
 public class ProviderIdSerializer extends Serializer<ProviderId> {
 
     /**
-     * Default constructor.
+     * Creates {@link ProviderId} serializer instance.
      */
     public ProviderIdSerializer() {
         // non-null, immutable
@@ -24,13 +24,15 @@ public class ProviderIdSerializer extends Serializer<ProviderId> {
     public void write(Kryo kryo, Output output, ProviderId object) {
         output.writeString(object.scheme());
         output.writeString(object.id());
+        output.writeBoolean(object.isAncillary());
     }
 
     @Override
     public ProviderId read(Kryo kryo, Input input, Class<ProviderId> type) {
         String scheme = input.readString();
         String id = input.readString();
-        return new ProviderId(scheme, id);
+        boolean isAncillary = input.readBoolean();
+        return new ProviderId(scheme, id, isAncillary);
     }
 
 }

@@ -6,41 +6,37 @@ import java.nio.ByteBuffer;
 /**
  * Service to serialize Objects into byte array.
  */
-public interface KryoSerializationService {
+public interface StoreSerializer {
 
     /**
-     * Serializes the specified object into bytes using one of the
-     * pre-registered serializers.
+     * Serializes the specified object into bytes.
      *
      * @param obj object to be serialized
      * @return serialized bytes
      */
-    public byte[] serialize(final Object obj);
+    public byte[] encode(final Object obj);
 
     /**
-     * Serializes the specified object into bytes using one of the
-     * pre-registered serializers.
+     * Serializes the specified object into bytes.
      *
      * @param obj object to be serialized
      * @param buffer to write serialized bytes
      */
-    public void serialize(final Object obj, ByteBuffer buffer);
+    public void encode(final Object obj, ByteBuffer buffer);
 
     /**
-     * Deserializes the specified bytes into an object using one of the
-     * pre-registered serializers.
+     * Deserializes the specified bytes into an object.
      *
      * @param bytes bytes to be deserialized
      * @return deserialized object
      */
-    public <T> T deserialize(final byte[] bytes);
+    public <T> T decode(final byte[] bytes);
 
     /**
-     * Deserializes the specified bytes into an object using one of the
-     * pre-registered serializers.
+     * Deserializes the specified bytes into an object.
      *
      * @param buffer bytes to be deserialized
      * @return deserialized object
      */
-    public <T> T deserialize(final ByteBuffer buffer);
+    public <T> T decode(final ByteBuffer buffer);
 }

 package org.onlab.onos.store.serializers;
 
+import static org.junit.Assert.assertEquals;
 import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.onlab.onos.net.PortNumber.portNumber;
 
-import java.net.URI;
 import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.HashMap;
 
 import org.junit.After;
 import org.junit.Before;
@@ -14,7 +12,9 @@ import org.junit.BeforeClass;
 import org.junit.Test;
 import org.onlab.onos.cluster.MastershipTerm;
 import org.onlab.onos.cluster.NodeId;
+import org.onlab.onos.net.Annotations;
 import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DefaultAnnotations;
 import org.onlab.onos.net.DefaultDevice;
 import org.onlab.onos.net.DefaultLink;
 import org.onlab.onos.net.DefaultPort;
@@ -24,7 +24,9 @@ import org.onlab.onos.net.Link;
 import org.onlab.onos.net.LinkKey;
 import org.onlab.onos.net.MastershipRole;
 import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.SparseAnnotations;
 import org.onlab.onos.net.provider.ProviderId;
+import org.onlab.packet.IpAddress;
 import org.onlab.packet.IpPrefix;
 import org.onlab.util.KryoPool;
 
@@ -32,10 +34,10 @@ import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.testing.EqualsTester;
 
-import de.javakaffee.kryoserializers.URISerializer;
+public class KryoSerializerTest {
 
-public class KryoSerializerTests {
     private static final ProviderId PID = new ProviderId("of", "foo");
+    private static final ProviderId PIDA = new ProviderId("of", "foo", true);
     private static final DeviceId DID1 = deviceId("of:foo");
     private static final DeviceId DID2 = deviceId("of:bar");
     private static final PortNumber P1 = portNumber(1);
@@ -48,44 +50,23 @@ public class KryoSerializerTests {
     private static final String SW2 = "3.9.5";
     private static final String SN = "43311-12345";
     private static final Device DEV1 = new DefaultDevice(PID, DID1, Device.Type.SWITCH, MFR, HW, SW1, SN);
+    private static final SparseAnnotations A1 = DefaultAnnotations.builder()
+            .set("A1", "a1")
+            .set("B1", "b1")
+            .build();
+    private static final SparseAnnotations A1_2 = DefaultAnnotations.builder()
+            .remove("A1")
+            .set("B3", "b3")
+            .build();
 
     private static KryoPool kryos;
 
     @BeforeClass
     public static void setUpBeforeClass() throws Exception {
         kryos = KryoPool.newBuilder()
-                .register(
-                        ArrayList.class,
-                        HashMap.class
-                        )
-                .register(
-                        Device.Type.class,
-                        Link.Type.class
-
-//                      ControllerNode.State.class,
-//                        DefaultControllerNode.class,
-//                        MastershipRole.class,
-//                        Port.class,
-//                        Element.class,
-                        )
-                .register(ConnectPoint.class, new ConnectPointSerializer())
-                .register(DefaultLink.class, new DefaultLinkSerializer())
-                .register(DefaultPort.class, new DefaultPortSerializer())
-                .register(DeviceId.class, new DeviceIdSerializer())
+                .register(KryoPoolUtil.API)
                 .register(ImmutableMap.class, new ImmutableMapSerializer())
                 .register(ImmutableSet.class, new ImmutableSetSerializer())
-                .register(IpPrefix.class, new IpPrefixSerializer())
-                .register(LinkKey.class, new LinkKeySerializer())
-                .register(NodeId.class, new NodeIdSerializer())
-                .register(PortNumber.class, new PortNumberSerializer())
-                .register(ProviderId.class, new ProviderIdSerializer())
-
-                .register(DefaultDevice.class)
-
-                .register(URI.class, new URISerializer())
-
-                .register(MastershipRole.class, new MastershipRoleSerializer())
-                .register(MastershipTerm.class, new MastershipTermSerializer())
                 .build();
     }
 
@@ -112,10 +93,12 @@ public class KryoSerializerTests {
 
 
     @Test
-    public final void test() {
+    public final void testSerialization() {
         testSerialized(new ConnectPoint(DID1, P1));
         testSerialized(new DefaultLink(PID, CP1, CP2, Link.Type.DIRECT));
         testSerialized(new DefaultPort(DEV1, P1, true));
+        testSerialized(new DefaultLink(PID, CP1, CP2, Link.Type.DIRECT, A1));
+        testSerialized(new DefaultPort(DEV1, P1, true, A1_2));
         testSerialized(DID1);
         testSerialized(ImmutableMap.of(DID1, DEV1, DID2, DEV1));
         testSerialized(ImmutableMap.of(DID1, DEV1));
@@ -124,10 +107,41 @@ public class KryoSerializerTests {
         testSerialized(ImmutableSet.of(DID1));
         testSerialized(ImmutableSet.of());
         testSerialized(IpPrefix.valueOf("192.168.0.1/24"));
+        testSerialized(IpAddress.valueOf("192.168.0.1"));
         testSerialized(new LinkKey(CP1, CP2));
         testSerialized(new NodeId("SomeNodeIdentifier"));
         testSerialized(P1);
         testSerialized(PID);
+        testSerialized(PIDA);
+        testSerialized(new NodeId("bar"));
+        testSerialized(MastershipTerm.of(new NodeId("foo"), 2));
+        for (MastershipRole role : MastershipRole.values()) {
+            testSerialized(role);
+        }
+    }
+
+    @Test
+    public final void testAnnotations() {
+        // Annotations does not have equals defined, manually test equality
+        final byte[] a1Bytes = kryos.serialize(A1);
+        SparseAnnotations copiedA1 = kryos.deserialize(a1Bytes);
+        assertAnnotationsEquals(copiedA1, A1);
+
+        final byte[] a12Bytes = kryos.serialize(A1_2);
+        SparseAnnotations copiedA12 = kryos.deserialize(a12Bytes);
+        assertAnnotationsEquals(copiedA12, A1_2);
+    }
+
+    // code clone
+    public static void assertAnnotationsEquals(Annotations actual, SparseAnnotations... annotations) {
+        SparseAnnotations expected = DefaultAnnotations.builder().build();
+        for (SparseAnnotations a : annotations) {
+            expected = DefaultAnnotations.union(expected, a);
+        }
+        assertEquals(expected.keys(), actual.keys());
+        for (String key : expected.keys()) {
+            assertEquals(expected.value(key), actual.value(key));
+        }
     }
 
 }

@@ -4,27 +4,15 @@ import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.onos.cluster.MastershipTerm;
 import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.store.ClockService;
-import org.onlab.onos.store.Timestamp;
+import org.onlab.onos.store.ClockProviderService;
 
 //FIXME: Code clone in onos-core-trivial, onos-core-hz-net
 /**
- * Dummy implementation of {@link ClockService}.
+ * Dummy implementation of {@link ClockProviderService}.
  */
 @Component(immediate = true)
 @Service
-public class NoOpClockService implements ClockService {
-
-    @Override
-    public Timestamp getTimestamp(DeviceId deviceId) {
-        return new Timestamp() {
-
-            @Override
-            public int compareTo(Timestamp o) {
-                throw new IllegalStateException("Never expected to be used.");
-            }
-        };
-    }
+public class NoOpClockProviderService implements ClockProviderService {
 
     @Override
     public void setMastershipTerm(DeviceId deviceId, MastershipTerm term) {

@@ -2,6 +2,8 @@ package org.onlab.onos.store.trivial.impl;
 
 import com.google.common.collect.FluentIterable;
 import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
 
 import org.apache.commons.lang3.concurrent.ConcurrentException;
 import org.apache.commons.lang3.concurrent.ConcurrentInitializer;
@@ -9,7 +11,7 @@ import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
-import org.onlab.onos.net.Annotations;
+import org.onlab.onos.net.AnnotationsUtil;
 import org.onlab.onos.net.DefaultAnnotations;
 import org.onlab.onos.net.DefaultDevice;
 import org.onlab.onos.net.DefaultPort;
@@ -28,10 +30,10 @@ import org.onlab.onos.net.device.DeviceStoreDelegate;
 import org.onlab.onos.net.device.PortDescription;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.AbstractStore;
+import org.onlab.util.NewConcurrentHashMap;
 import org.slf4j.Logger;
 
 import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Iterator;
@@ -47,12 +49,13 @@ import java.util.concurrent.atomic.AtomicReference;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
 import static com.google.common.base.Predicates.notNull;
+import static com.google.common.base.Verify.verify;
 import static org.onlab.onos.net.device.DeviceEvent.Type.*;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.apache.commons.lang3.concurrent.ConcurrentUtils.createIfAbsentUnchecked;
+import static org.onlab.onos.net.DefaultAnnotations.union;
 import static org.onlab.onos.net.DefaultAnnotations.merge;
 
-// TODO: synchronization should be done in more fine-grained manner.
 /**
  * Manages inventory of infrastructure devices using trivial in-memory
  * structures implementation.
@@ -70,14 +73,14 @@ public class SimpleDeviceStore
     // collection of Description given from various providers
     private final ConcurrentMap<DeviceId,
                             ConcurrentMap<ProviderId, DeviceDescriptions>>
-                                deviceDescs = new ConcurrentHashMap<>();
+                                deviceDescs = Maps.newConcurrentMap();
 
     // cache of Device and Ports generated by compositing descriptions from providers
-    private final ConcurrentMap<DeviceId, Device> devices = new ConcurrentHashMap<>();
-    private final ConcurrentMap<DeviceId, ConcurrentMap<PortNumber, Port>> devicePorts = new ConcurrentHashMap<>();
+    private final ConcurrentMap<DeviceId, Device> devices = Maps.newConcurrentMap();
+    private final ConcurrentMap<DeviceId, ConcurrentMap<PortNumber, Port>> devicePorts = Maps.newConcurrentMap();
 
     // available(=UP) devices
-    private final Set<DeviceId> availableDevices = new HashSet<>();
+    private final Set<DeviceId> availableDevices = Sets.newConcurrentHashSet();
 
 
     @Activate
@@ -87,6 +90,10 @@ public class SimpleDeviceStore
 
     @Deactivate
     public void deactivate() {
+        deviceDescs.clear();
+        devices.clear();
+        devicePorts.clear();
+        availableDevices.clear();
         log.info("Stopped");
     }
 
@@ -106,117 +113,142 @@ public class SimpleDeviceStore
     }
 
     @Override
-    public synchronized DeviceEvent createOrUpdateDevice(ProviderId providerId, DeviceId deviceId,
+    public DeviceEvent createOrUpdateDevice(ProviderId providerId,
+                                     DeviceId deviceId,
                                      DeviceDescription deviceDescription) {
+
         ConcurrentMap<ProviderId, DeviceDescriptions> providerDescs
-            = createIfAbsentUnchecked(deviceDescs, deviceId,
-                    new InitConcurrentHashMap<ProviderId, DeviceDescriptions>());
+            = getDeviceDescriptions(deviceId);
 
-        Device oldDevice = devices.get(deviceId);
+        synchronized (providerDescs) {
+            // locking per device
 
-        DeviceDescriptions descs
-            = createIfAbsentUnchecked(providerDescs, providerId,
-                    new InitDeviceDescs(deviceDescription));
+            DeviceDescriptions descs
+                = createIfAbsentUnchecked(providerDescs, providerId,
+                        new InitDeviceDescs(deviceDescription));
 
-        // update description
-        descs.putDeviceDesc(deviceDescription);
-        Device newDevice = composeDevice(deviceId, providerDescs);
+            Device oldDevice = devices.get(deviceId);
+            // update description
+            descs.putDeviceDesc(deviceDescription);
+            Device newDevice = composeDevice(deviceId, providerDescs);
 
-        if (oldDevice == null) {
-            // ADD
-            return createDevice(providerId, newDevice);
-        } else {
-            // UPDATE or ignore (no change or stale)
-            return updateDevice(providerId, oldDevice, newDevice);
+            if (oldDevice == null) {
+                // ADD
+                return createDevice(providerId, newDevice);
+            } else {
+                // UPDATE or ignore (no change or stale)
+                return updateDevice(providerId, oldDevice, newDevice);
+            }
         }
     }
 
     // Creates the device and returns the appropriate event if necessary.
+    // Guarded by deviceDescs value (=Device lock)
     private DeviceEvent createDevice(ProviderId providerId, Device newDevice) {
 
         // update composed device cache
-        synchronized (this) {
-            devices.putIfAbsent(newDevice.id(), newDevice);
-            if (!providerId.isAncillary()) {
-                availableDevices.add(newDevice.id());
-            }
+        Device oldDevice = devices.putIfAbsent(newDevice.id(), newDevice);
+        verify(oldDevice == null,
+                "Unexpected Device in cache. PID:%s [old=%s, new=%s]",
+                providerId, oldDevice, newDevice);
+
+        if (!providerId.isAncillary()) {
+            availableDevices.add(newDevice.id());
         }
 
         return new DeviceEvent(DeviceEvent.Type.DEVICE_ADDED, newDevice, null);
     }
 
     // Updates the device and returns the appropriate event if necessary.
+    // Guarded by deviceDescs value (=Device lock)
     private DeviceEvent updateDevice(ProviderId providerId, Device oldDevice, Device newDevice) {
 
         // We allow only certain attributes to trigger update
         if (!Objects.equals(oldDevice.hwVersion(), newDevice.hwVersion()) ||
             !Objects.equals(oldDevice.swVersion(), newDevice.swVersion()) ||
-            !isAnnotationsEqual(oldDevice.annotations(), newDevice.annotations())) {
-
-            synchronized (this) {
-                devices.replace(newDevice.id(), oldDevice, newDevice);
-                if (!providerId.isAncillary()) {
-                    availableDevices.add(newDevice.id());
-                }
+            !AnnotationsUtil.isEqual(oldDevice.annotations(), newDevice.annotations())) {
+
+            boolean replaced = devices.replace(newDevice.id(), oldDevice, newDevice);
+            if (!replaced) {
+                verify(replaced,
+                        "Replacing devices cache failed. PID:%s [expected:%s, found:%s, new=%s]",
+                        providerId, oldDevice, devices.get(newDevice.id())
+                        , newDevice);
+            }
+            if (!providerId.isAncillary()) {
+                availableDevices.add(newDevice.id());
             }
             return new DeviceEvent(DeviceEvent.Type.DEVICE_UPDATED, newDevice, null);
         }
 
         // Otherwise merely attempt to change availability if primary provider
         if (!providerId.isAncillary()) {
-            synchronized (this) {
             boolean added = availableDevices.add(newDevice.id());
             return !added ? null :
                     new DeviceEvent(DEVICE_AVAILABILITY_CHANGED, newDevice, null);
-            }
         }
         return null;
     }
 
     @Override
     public DeviceEvent markOffline(DeviceId deviceId) {
-        synchronized (this) {
+        ConcurrentMap<ProviderId, DeviceDescriptions> providerDescs
+            = getDeviceDescriptions(deviceId);
+
+        // locking device
+        synchronized (providerDescs) {
             Device device = devices.get(deviceId);
-            boolean removed = (device != null) && availableDevices.remove(deviceId);
-            return !removed ? null :
-                    new DeviceEvent(DEVICE_AVAILABILITY_CHANGED, device, null);
+            if (device == null) {
+                return null;
+            }
+            boolean removed = availableDevices.remove(deviceId);
+            if (removed) {
+                // TODO: broadcast ... DOWN only?
+                return new DeviceEvent(DEVICE_AVAILABILITY_CHANGED, device, null);
+            }
+            return null;
         }
     }
 
     @Override
-    public synchronized List<DeviceEvent> updatePorts(ProviderId providerId, DeviceId deviceId,
-                                  List<PortDescription> portDescriptions) {
+    public List<DeviceEvent> updatePorts(ProviderId providerId,
+                                      DeviceId deviceId,
+                                      List<PortDescription> portDescriptions) {
 
-        // TODO: implement multi-provider
         Device device = devices.get(deviceId);
         checkArgument(device != null, DEVICE_NOT_FOUND, deviceId);
 
         ConcurrentMap<ProviderId, DeviceDescriptions> descsMap = deviceDescs.get(deviceId);
         checkArgument(descsMap != null, DEVICE_NOT_FOUND, deviceId);
 
-        DeviceDescriptions descs = descsMap.get(providerId);
-        checkArgument(descs != null,
-                "Device description for Device ID %s from Provider %s was not found",
-                deviceId, providerId);
-
-
         List<DeviceEvent> events = new ArrayList<>();
-        synchronized (this) {
-            ConcurrentMap<PortNumber, Port> ports = getPortMap(deviceId);
+        synchronized (descsMap) {
+            DeviceDescriptions descs = descsMap.get(providerId);
+            // every provider must provide DeviceDescription.
+            checkArgument(descs != null,
+                    "Device description for Device ID %s from Provider %s was not found",
+                    deviceId, providerId);
+
+            Map<PortNumber, Port> ports = getPortMap(deviceId);
 
             // Add new ports
             Set<PortNumber> processed = new HashSet<>();
             for (PortDescription portDescription : portDescriptions) {
-                PortNumber number = portDescription.portNumber();
-                Port oldPort = ports.get(number);
+                final PortNumber number = portDescription.portNumber();
+                processed.add(portDescription.portNumber());
+
+                final Port oldPort = ports.get(number);
+                final Port newPort;
+
+// event suppression hook?
+
                 // update description
                 descs.putPortDesc(portDescription);
-                Port newPort = composePort(device, number, descsMap);
+                newPort = composePort(device, number, descsMap);
 
                 events.add(oldPort == null ?
-                                   createPort(device, newPort, ports) :
-                                   updatePort(device, oldPort, newPort, ports));
-                processed.add(portDescription.portNumber());
+                        createPort(device, newPort, ports) :
+                        updatePort(device, oldPort, newPort, ports));
             }
 
             events.addAll(pruneOldPorts(device, ports, processed));
@@ -226,19 +258,21 @@ public class SimpleDeviceStore
 
     // Creates a new port based on the port description adds it to the map and
     // Returns corresponding event.
+    // Guarded by deviceDescs value (=Device lock)
     private DeviceEvent createPort(Device device, Port newPort,
-                                   ConcurrentMap<PortNumber, Port> ports) {
+                                   Map<PortNumber, Port> ports) {
         ports.put(newPort.number(), newPort);
         return new DeviceEvent(PORT_ADDED, device, newPort);
     }
 
     // Checks if the specified port requires update and if so, it replaces the
     // existing entry in the map and returns corresponding event.
+    // Guarded by deviceDescs value (=Device lock)
     private DeviceEvent updatePort(Device device, Port oldPort,
                                    Port newPort,
-                                   ConcurrentMap<PortNumber, Port> ports) {
+                                   Map<PortNumber, Port> ports) {
         if (oldPort.isEnabled() != newPort.isEnabled() ||
-            !isAnnotationsEqual(oldPort.annotations(), newPort.annotations())) {
+            !AnnotationsUtil.isEqual(oldPort.annotations(), newPort.annotations())) {
 
             ports.put(oldPort.number(), newPort);
             return new DeviceEvent(PORT_UPDATED, device, newPort);
@@ -248,6 +282,7 @@ public class SimpleDeviceStore
 
     // Prunes the specified list of ports based on which ports are in the
     // processed list and returns list of corresponding events.
+    // Guarded by deviceDescs value (=Device lock)
     private List<DeviceEvent> pruneOldPorts(Device device,
                                             Map<PortNumber, Port> ports,
                                             Set<PortNumber> processed) {
@@ -268,11 +303,17 @@ public class SimpleDeviceStore
     // exist, it creates and registers a new one.
     private ConcurrentMap<PortNumber, Port> getPortMap(DeviceId deviceId) {
         return createIfAbsentUnchecked(devicePorts, deviceId,
-                new InitConcurrentHashMap<PortNumber, Port>());
+                NewConcurrentHashMap.<PortNumber, Port>ifNeeded());
+    }
+
+    private ConcurrentMap<ProviderId, DeviceDescriptions> getDeviceDescriptions(
+            DeviceId deviceId) {
+        return createIfAbsentUnchecked(deviceDescs, deviceId,
+                NewConcurrentHashMap.<ProviderId, DeviceDescriptions>ifNeeded());
     }
 
     @Override
-    public synchronized DeviceEvent updatePortStatus(ProviderId providerId, DeviceId deviceId,
+    public DeviceEvent updatePortStatus(ProviderId providerId, DeviceId deviceId,
                                  PortDescription portDescription) {
         Device device = devices.get(deviceId);
         checkArgument(device != null, DEVICE_NOT_FOUND, deviceId);
@@ -280,19 +321,22 @@ public class SimpleDeviceStore
         ConcurrentMap<ProviderId, DeviceDescriptions> descsMap = deviceDescs.get(deviceId);
         checkArgument(descsMap != null, DEVICE_NOT_FOUND, deviceId);
 
-        DeviceDescriptions descs = descsMap.get(providerId);
-        // assuming all providers must to give DeviceDescription
-        checkArgument(descs != null,
-                "Device description for Device ID %s from Provider %s was not found",
-                deviceId, providerId);
+        synchronized (descsMap) {
+            DeviceDescriptions descs = descsMap.get(providerId);
+            // assuming all providers must to give DeviceDescription
+            checkArgument(descs != null,
+                    "Device description for Device ID %s from Provider %s was not found",
+                    deviceId, providerId);
 
-        synchronized (this) {
             ConcurrentMap<PortNumber, Port> ports = getPortMap(deviceId);
             final PortNumber number = portDescription.portNumber();
-            Port oldPort = ports.get(number);
+            final Port oldPort = ports.get(number);
+            final Port newPort;
+
             // update description
             descs.putPortDesc(portDescription);
-            Port newPort = composePort(device, number, descsMap);
+            newPort = composePort(device, number, descsMap);
+
             if (oldPort == null) {
                 return createPort(device, newPort, ports);
             } else {
@@ -323,31 +367,19 @@ public class SimpleDeviceStore
 
     @Override
     public DeviceEvent removeDevice(DeviceId deviceId) {
-        synchronized (this) {
+        ConcurrentMap<ProviderId, DeviceDescriptions> descs = getDeviceDescriptions(deviceId);
+        synchronized (descs) {
             Device device = devices.remove(deviceId);
-            return device == null ? null :
-                    new DeviceEvent(DEVICE_REMOVED, device, null);
-        }
-    }
-
-    private static boolean isAnnotationsEqual(Annotations lhs, Annotations rhs) {
-        if (lhs == rhs) {
-            return true;
-        }
-        if (lhs == null || rhs == null) {
-            return false;
-        }
-
-        if (!lhs.keys().equals(rhs.keys())) {
-            return false;
-        }
-
-        for (String key : lhs.keys()) {
-            if (!lhs.value(key).equals(rhs.value(key))) {
-                return false;
+            // should DEVICE_REMOVED carry removed ports?
+            Map<PortNumber, Port> ports = devicePorts.get(deviceId);
+            if (ports != null) {
+                ports.clear();
             }
+            availableDevices.remove(deviceId);
+            descs.clear();
+            return device == null ? null :
+                new DeviceEvent(DEVICE_REMOVED, device, null);
         }
-        return true;
     }
 
     /**
@@ -366,14 +398,14 @@ public class SimpleDeviceStore
 
         DeviceDescriptions desc = providerDescs.get(primary);
 
-        // base
-        Type type = desc.getDeviceDesc().type();
-        String manufacturer = desc.getDeviceDesc().manufacturer();
-        String hwVersion = desc.getDeviceDesc().hwVersion();
-        String swVersion = desc.getDeviceDesc().swVersion();
-        String serialNumber = desc.getDeviceDesc().serialNumber();
+        final DeviceDescription base = desc.getDeviceDesc();
+        Type type = base.type();
+        String manufacturer = base.manufacturer();
+        String hwVersion = base.hwVersion();
+        String swVersion = base.swVersion();
+        String serialNumber = base.serialNumber();
         DefaultAnnotations annotations = DefaultAnnotations.builder().build();
-        annotations = merge(annotations, desc.getDeviceDesc().annotations());
+        annotations = merge(annotations, base.annotations());
 
         for (Entry<ProviderId, DeviceDescriptions> e : providerDescs.entrySet()) {
             if (e.getKey().equals(primary)) {
@@ -392,7 +424,14 @@ public class SimpleDeviceStore
                             hwVersion, swVersion, serialNumber, annotations);
     }
 
-    // probably want composePort"s" also
+    /**
+     * Returns a Port, merging description given from multiple Providers.
+     *
+     * @param device device the port is on
+     * @param number port number
+     * @param providerDescs Collection of Descriptions from multiple providers
+     * @return Port instance
+     */
     private Port composePort(Device device, PortNumber number,
                 ConcurrentMap<ProviderId, DeviceDescriptions> providerDescs) {
 
@@ -445,18 +484,11 @@ public class SimpleDeviceStore
         return fallBackPrimary;
     }
 
-    // TODO: can be made generic
-    private static final class InitConcurrentHashMap<K, V> implements
-            ConcurrentInitializer<ConcurrentMap<K, V>> {
-        @Override
-        public ConcurrentMap<K, V> get() throws ConcurrentException {
-            return new ConcurrentHashMap<>();
-        }
-    }
-
     public static final class InitDeviceDescs
         implements ConcurrentInitializer<DeviceDescriptions> {
+
         private final DeviceDescription deviceDesc;
+
         public InitDeviceDescs(DeviceDescription deviceDesc) {
             this.deviceDesc = checkNotNull(deviceDesc);
         }
@@ -471,8 +503,6 @@ public class SimpleDeviceStore
      * Collection of Description of a Device and it's Ports given from a Provider.
      */
     private static class DeviceDescriptions {
-        //        private final DeviceId id;
-        //        private final ProviderId pid;
 
         private final AtomicReference<DeviceDescription> deviceDesc;
         private final ConcurrentMap<PortNumber, PortDescription> portDescs;
@@ -490,10 +520,6 @@ public class SimpleDeviceStore
             return portDescs.get(number);
         }
 
-        public Collection<PortDescription> getPortDescs() {
-            return Collections.unmodifiableCollection(portDescs.values());
-        }
-
         /**
          * Puts DeviceDescription, merging annotations as necessary.
          *
@@ -504,7 +530,7 @@ public class SimpleDeviceStore
             DeviceDescription oldOne = deviceDesc.get();
             DeviceDescription newOne = newDesc;
             if (oldOne != null) {
-                SparseAnnotations merged = merge(oldOne.annotations(),
+                SparseAnnotations merged = union(oldOne.annotations(),
                                                  newDesc.annotations());
                 newOne = new DefaultDeviceDescription(newOne, merged);
             }
@@ -521,7 +547,7 @@ public class SimpleDeviceStore
             PortDescription oldOne = portDescs.get(newDesc.portNumber());
             PortDescription newOne = newDesc;
             if (oldOne != null) {
-                SparseAnnotations merged = merge(oldOne.annotations(),
+                SparseAnnotations merged = union(oldOne.annotations(),
                                                  newDesc.annotations());
                 newOne = new DefaultPortDescription(newOne, merged);
             }

@@ -12,9 +12,10 @@ import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.net.DeviceId;
-import org.onlab.onos.net.flow.DefaultFlowRule;
+import org.onlab.onos.net.flow.DefaultFlowEntry;
+import org.onlab.onos.net.flow.FlowEntry;
+import org.onlab.onos.net.flow.FlowEntry.FlowEntryState;
 import org.onlab.onos.net.flow.FlowRule;
-import org.onlab.onos.net.flow.FlowRule.FlowRuleState;
 import org.onlab.onos.net.flow.FlowRuleEvent;
 import org.onlab.onos.net.flow.FlowRuleEvent.Type;
 import org.onlab.onos.net.flow.FlowRuleStore;
@@ -38,8 +39,8 @@ public class SimpleFlowRuleStore
     private final Logger log = getLogger(getClass());
 
     // store entries as a pile of rules, no info about device tables
-    private final Multimap<DeviceId, FlowRule> flowEntries =
-            ArrayListMultimap.<DeviceId, FlowRule>create();
+    private final Multimap<DeviceId, FlowEntry> flowEntries =
+            ArrayListMultimap.<DeviceId, FlowEntry>create();
 
     private final Multimap<ApplicationId, FlowRule> flowEntriesById =
             ArrayListMultimap.<ApplicationId, FlowRule>create();
@@ -56,8 +57,13 @@ public class SimpleFlowRuleStore
 
 
     @Override
-    public synchronized FlowRule getFlowRule(FlowRule rule) {
-        for (FlowRule f : flowEntries.get(rule.deviceId())) {
+    public int getFlowRuleCount() {
+        return flowEntries.size();
+    }
+
+    @Override
+    public synchronized FlowEntry getFlowEntry(FlowRule rule) {
+        for (FlowEntry f : flowEntries.get(rule.deviceId())) {
             if (f.equals(rule)) {
                 return f;
             }
@@ -66,8 +72,8 @@ public class SimpleFlowRuleStore
     }
 
     @Override
-    public synchronized Iterable<FlowRule> getFlowEntries(DeviceId deviceId) {
-        Collection<FlowRule> rules = flowEntries.get(deviceId);
+    public synchronized Iterable<FlowEntry> getFlowEntries(DeviceId deviceId) {
+        Collection<FlowEntry> rules = flowEntries.get(deviceId);
         if (rules == null) {
             return Collections.emptyList();
         }
@@ -75,7 +81,7 @@ public class SimpleFlowRuleStore
     }
 
     @Override
-    public synchronized Iterable<FlowRule> getFlowEntriesByAppId(ApplicationId appId) {
+    public synchronized Iterable<FlowRule> getFlowRulesByAppId(ApplicationId appId) {
         Collection<FlowRule> rules = flowEntriesById.get(appId);
         if (rules == null) {
             return Collections.emptyList();
@@ -85,7 +91,7 @@ public class SimpleFlowRuleStore
 
     @Override
     public synchronized void storeFlowRule(FlowRule rule) {
-        FlowRule f = new DefaultFlowRule(rule, FlowRuleState.PENDING_ADD);
+        FlowEntry f = new DefaultFlowEntry(rule);
         DeviceId did = f.deviceId();
         if (!flowEntries.containsEntry(did, f)) {
             flowEntries.put(did, f);
@@ -95,51 +101,42 @@ public class SimpleFlowRuleStore
 
     @Override
     public synchronized void deleteFlowRule(FlowRule rule) {
-        FlowRule f = new DefaultFlowRule(rule, FlowRuleState.PENDING_REMOVE);
-        DeviceId did = f.deviceId();
-
-        /*
-         *  find the rule and mark it for deletion.
-         *  Ultimately a flow removed will come remove it.
-         */
-
-        if (flowEntries.containsEntry(did, f)) {
-            flowEntries.remove(did, f);
-            flowEntries.put(did, f);
-            flowEntriesById.remove(rule.appId(), rule);
+        FlowEntry entry = getFlowEntry(rule);
+        if (entry == null) {
+            //log.warn("Cannot find rule {}", rule);
+            return;
         }
+        entry.setState(FlowEntryState.PENDING_REMOVE);
     }
 
     @Override
-    public synchronized FlowRuleEvent addOrUpdateFlowRule(FlowRule rule) {
+    public synchronized FlowRuleEvent addOrUpdateFlowRule(FlowEntry rule) {
         DeviceId did = rule.deviceId();
 
         // check if this new rule is an update to an existing entry
-        FlowRule stored = getFlowRule(rule);
+        FlowEntry stored = getFlowEntry(rule);
         if (stored != null) {
-            // Multimaps support duplicates so we have to remove our rule
-            // and replace it with the current version.
-            flowEntries.remove(did, rule);
-            flowEntries.put(did, rule);
-
-            if (stored.state() == FlowRuleState.PENDING_ADD) {
+            stored.setBytes(rule.bytes());
+            stored.setLife(rule.life());
+            stored.setPackets(rule.packets());
+            if (stored.state() == FlowEntryState.PENDING_ADD) {
+                stored.setState(FlowEntryState.ADDED);
                 return new FlowRuleEvent(Type.RULE_ADDED, rule);
             }
             return new FlowRuleEvent(Type.RULE_UPDATED, rule);
         }
 
-        flowEntries.put(did, rule);
+        //flowEntries.put(did, rule);
         return null;
     }
 
     @Override
-    public synchronized FlowRuleEvent removeFlowRule(FlowRule rule) {
-        //synchronized (this) {
+    public synchronized FlowRuleEvent removeFlowRule(FlowEntry rule) {
+        // This is where one could mark a rule as removed and still keep it in the store.
         if (flowEntries.remove(rule.deviceId(), rule)) {
             return new FlowRuleEvent(RULE_REMOVED, rule);
         } else {
             return null;
         }
-        //}
     }
 }

+package org.onlab.onos.store.trivial.impl;
+
+import com.google.common.collect.ImmutableSet;
+import org.apache.felix.scr.annotations.Activate;
+import org.apache.felix.scr.annotations.Component;
+import org.apache.felix.scr.annotations.Deactivate;
+import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.net.intent.InstallableIntent;
+import org.onlab.onos.net.intent.Intent;
+import org.onlab.onos.net.intent.IntentEvent;
+import org.onlab.onos.net.intent.IntentId;
+import org.onlab.onos.net.intent.IntentState;
+import org.onlab.onos.net.intent.IntentStore;
+import org.onlab.onos.net.intent.IntentStoreDelegate;
+import org.onlab.onos.store.AbstractStore;
+import org.slf4j.Logger;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.onlab.onos.net.intent.IntentState.*;
+import static org.slf4j.LoggerFactory.getLogger;
+
+@Component(immediate = true)
+@Service
+public class SimpleIntentStore
+        extends AbstractStore<IntentEvent, IntentStoreDelegate>
+        implements IntentStore {
+
+    private final Logger log = getLogger(getClass());
+    private final Map<IntentId, Intent> intents = new HashMap<>();
+    private final Map<IntentId, IntentState> states = new HashMap<>();
+    private final Map<IntentId, List<InstallableIntent>> installable = new HashMap<>();
+
+    @Activate
+    public void activate() {
+        log.info("Started");
+    }
+
+    @Deactivate
+    public void deactivate() {
+        log.info("Stopped");
+    }
+
+    @Override
+    public IntentEvent createIntent(Intent intent) {
+        intents.put(intent.id(), intent);
+        return this.setState(intent, IntentState.SUBMITTED);
+    }
+
+    @Override
+    public IntentEvent removeIntent(IntentId intentId) {
+        Intent intent = intents.remove(intentId);
+        installable.remove(intentId);
+        IntentEvent event = this.setState(intent, WITHDRAWN);
+        states.remove(intentId);
+        return event;
+    }
+
+    @Override
+    public long getIntentCount() {
+        return intents.size();
+    }
+
+    @Override
+    public Iterable<Intent> getIntents() {
+        return ImmutableSet.copyOf(intents.values());
+    }
+
+    @Override
+    public Intent getIntent(IntentId intentId) {
+        return intents.get(intentId);
+    }
+
+    @Override
+    public IntentState getIntentState(IntentId id) {
+        return states.get(id);
+    }
+
+    @Override
+    public IntentEvent setState(Intent intent, IntentState state) {
+        IntentId id = intent.id();
+        states.put(id, state);
+        IntentEvent.Type type = (state == SUBMITTED ? IntentEvent.Type.SUBMITTED :
+                (state == INSTALLED ? IntentEvent.Type.INSTALLED :
+                        (state == FAILED ? IntentEvent.Type.FAILED :
+                                state == WITHDRAWN ? IntentEvent.Type.WITHDRAWN :
+                                        null)));
+        return type == null ? null : new IntentEvent(type, intent);
+    }
+
+    @Override
+    public void addInstallableIntents(IntentId intentId, List<InstallableIntent> result) {
+        installable.put(intentId, result);
+    }
+
+    @Override
+    public List<InstallableIntent> getInstallableIntents(IntentId intentId) {
+        return installable.get(intentId);
+    }
+
+    @Override
+    public void removeInstalledIntents(IntentId intentId) {
+        installable.remove(intentId);
+    }
+
+}

 package org.onlab.onos.store.trivial.impl;
 
+import com.google.common.base.Function;
+import com.google.common.base.Predicate;
+import com.google.common.collect.FluentIterable;
 import com.google.common.collect.HashMultimap;
-import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Multimap;
+import com.google.common.collect.SetMultimap;
 
+import org.apache.commons.lang3.concurrent.ConcurrentUtils;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
 import org.apache.felix.scr.annotations.Service;
+import org.onlab.onos.net.AnnotationsUtil;
 import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DefaultAnnotations;
 import org.onlab.onos.net.DefaultLink;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Link;
+import org.onlab.onos.net.SparseAnnotations;
+import org.onlab.onos.net.Link.Type;
 import org.onlab.onos.net.LinkKey;
+import org.onlab.onos.net.Provided;
+import org.onlab.onos.net.link.DefaultLinkDescription;
 import org.onlab.onos.net.link.LinkDescription;
 import org.onlab.onos.net.link.LinkEvent;
 import org.onlab.onos.net.link.LinkStore;
 import org.onlab.onos.net.link.LinkStoreDelegate;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.store.AbstractStore;
+import org.onlab.util.NewConcurrentHashMap;
 import org.slf4j.Logger;
 
 import java.util.Collections;
 import java.util.HashSet;
-import java.util.Map;
 import java.util.Set;
+import java.util.Map.Entry;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
 
+import static org.onlab.onos.net.DefaultAnnotations.union;
+import static org.onlab.onos.net.DefaultAnnotations.merge;
 import static org.onlab.onos.net.Link.Type.DIRECT;
 import static org.onlab.onos.net.Link.Type.INDIRECT;
 import static org.onlab.onos.net.link.LinkEvent.Type.*;
 import static org.slf4j.LoggerFactory.getLogger;
+import static com.google.common.collect.Multimaps.synchronizedSetMultimap;
+import static com.google.common.base.Predicates.notNull;
 
 /**
  * Manages inventory of infrastructure links using trivial in-memory structures
@@ -45,11 +60,17 @@ public class SimpleLinkStore
     private final Logger log = getLogger(getClass());
 
     // Link inventory
-    private final Map<LinkKey, DefaultLink> links = new ConcurrentHashMap<>();
+    private final ConcurrentMap<LinkKey,
+            ConcurrentMap<ProviderId, LinkDescription>>
+                    linkDescs = new ConcurrentHashMap<>();
+
+    // Link instance cache
+    private final ConcurrentMap<LinkKey, Link> links = new ConcurrentHashMap<>();
 
     // Egress and ingress link sets
-    private final Multimap<DeviceId, Link> srcLinks = HashMultimap.create();
-    private final Multimap<DeviceId, Link> dstLinks = HashMultimap.create();
+    private final SetMultimap<DeviceId, LinkKey> srcLinks = createSynchronizedHashMultiMap();
+    private final SetMultimap<DeviceId, LinkKey> dstLinks = createSynchronizedHashMultiMap();
+
 
     @Activate
     public void activate() {
@@ -58,6 +79,10 @@ public class SimpleLinkStore
 
     @Deactivate
     public void deactivate() {
+        linkDescs.clear();
+        links.clear();
+        srcLinks.clear();
+        dstLinks.clear();
         log.info("Stopped");
     }
 
@@ -68,17 +93,29 @@ public class SimpleLinkStore
 
     @Override
     public Iterable<Link> getLinks() {
-        return Collections.unmodifiableSet(new HashSet<Link>(links.values()));
+        return Collections.unmodifiableCollection(links.values());
     }
 
     @Override
     public Set<Link> getDeviceEgressLinks(DeviceId deviceId) {
-        return ImmutableSet.copyOf(srcLinks.get(deviceId));
+        // lock for iteration
+        synchronized (srcLinks) {
+            return FluentIterable.from(srcLinks.get(deviceId))
+            .transform(lookupLink())
+            .filter(notNull())
+            .toSet();
+        }
     }
 
     @Override
     public Set<Link> getDeviceIngressLinks(DeviceId deviceId) {
-        return ImmutableSet.copyOf(dstLinks.get(deviceId));
+        // lock for iteration
+        synchronized (dstLinks) {
+            return FluentIterable.from(dstLinks.get(deviceId))
+            .transform(lookupLink())
+            .filter(notNull())
+            .toSet();
+        }
     }
 
     @Override
@@ -89,9 +126,9 @@ public class SimpleLinkStore
     @Override
     public Set<Link> getEgressLinks(ConnectPoint src) {
         Set<Link> egress = new HashSet<>();
-        for (Link link : srcLinks.get(src.deviceId())) {
-            if (link.src().equals(src)) {
-                egress.add(link);
+        for (LinkKey linkKey : srcLinks.get(src.deviceId())) {
+            if (linkKey.src().equals(src)) {
+                egress.add(links.get(linkKey));
             }
         }
         return egress;
@@ -100,9 +137,9 @@ public class SimpleLinkStore
     @Override
     public Set<Link> getIngressLinks(ConnectPoint dst) {
         Set<Link> ingress = new HashSet<>();
-        for (Link link : dstLinks.get(dst.deviceId())) {
-            if (link.dst().equals(dst)) {
-                ingress.add(link);
+        for (LinkKey linkKey : dstLinks.get(dst.deviceId())) {
+            if (linkKey.dst().equals(dst)) {
+                ingress.add(links.get(linkKey));
             }
         }
         return ingress;
@@ -112,56 +149,172 @@ public class SimpleLinkStore
     public LinkEvent createOrUpdateLink(ProviderId providerId,
                                         LinkDescription linkDescription) {
         LinkKey key = new LinkKey(linkDescription.src(), linkDescription.dst());
-        DefaultLink link = links.get(key);
-        if (link == null) {
-            return createLink(providerId, key, linkDescription);
+
+        ConcurrentMap<ProviderId, LinkDescription> descs = getLinkDescriptions(key);
+        synchronized (descs) {
+            final Link oldLink = links.get(key);
+            // update description
+            createOrUpdateLinkDescription(descs, providerId, linkDescription);
+            final Link newLink = composeLink(descs);
+            if (oldLink == null) {
+                return createLink(key, newLink);
+            }
+            return updateLink(key, oldLink, newLink);
         }
-        return updateLink(providerId, link, key, linkDescription);
+    }
+
+    // Guarded by linkDescs value (=locking each Link)
+    private LinkDescription createOrUpdateLinkDescription(
+                             ConcurrentMap<ProviderId, LinkDescription> descs,
+                             ProviderId providerId,
+                             LinkDescription linkDescription) {
+
+        // merge existing attributes and merge
+        LinkDescription oldDesc = descs.get(providerId);
+        LinkDescription newDesc = linkDescription;
+        if (oldDesc != null) {
+            SparseAnnotations merged = union(oldDesc.annotations(),
+                    linkDescription.annotations());
+            newDesc = new DefaultLinkDescription(
+                        linkDescription.src(),
+                        linkDescription.dst(),
+                        linkDescription.type(), merged);
+        }
+        return descs.put(providerId, newDesc);
     }
 
     // Creates and stores the link and returns the appropriate event.
-    private LinkEvent createLink(ProviderId providerId, LinkKey key,
-                                 LinkDescription linkDescription) {
-        DefaultLink link = new DefaultLink(providerId, key.src(), key.dst(),
-                                           linkDescription.type());
-        synchronized (this) {
-            links.put(key, link);
-            srcLinks.put(link.src().deviceId(), link);
-            dstLinks.put(link.dst().deviceId(), link);
+    // Guarded by linkDescs value (=locking each Link)
+    private LinkEvent createLink(LinkKey key, Link newLink) {
+
+        if (newLink.providerId().isAncillary()) {
+            // TODO: revisit ancillary only Link handling
+
+            // currently treating ancillary only as down (not visible outside)
+            return null;
         }
-        return new LinkEvent(LINK_ADDED, link);
+
+        links.put(key, newLink);
+        srcLinks.put(newLink.src().deviceId(), key);
+        dstLinks.put(newLink.dst().deviceId(), key);
+        return new LinkEvent(LINK_ADDED, newLink);
     }
 
     // Updates, if necessary the specified link and returns the appropriate event.
-    private LinkEvent updateLink(ProviderId providerId, DefaultLink link,
-                                 LinkKey key, LinkDescription linkDescription) {
-        if (link.type() == INDIRECT && linkDescription.type() == DIRECT) {
-            synchronized (this) {
-                srcLinks.remove(link.src().deviceId(), link);
-                dstLinks.remove(link.dst().deviceId(), link);
-
-                DefaultLink updated =
-                        new DefaultLink(providerId, link.src(), link.dst(),
-                                        linkDescription.type());
-                links.put(key, updated);
-                srcLinks.put(link.src().deviceId(), updated);
-                dstLinks.put(link.dst().deviceId(), updated);
-                return new LinkEvent(LINK_UPDATED, updated);
-            }
+    // Guarded by linkDescs value (=locking each Link)
+    private LinkEvent updateLink(LinkKey key, Link oldLink, Link newLink) {
+
+        if (newLink.providerId().isAncillary()) {
+            // TODO: revisit ancillary only Link handling
+
+            // currently treating ancillary only as down (not visible outside)
+            return null;
+        }
+
+        if ((oldLink.type() == INDIRECT && newLink.type() == DIRECT) ||
+            !AnnotationsUtil.isEqual(oldLink.annotations(), newLink.annotations())) {
+
+            links.put(key, newLink);
+            // strictly speaking following can be ommitted
+            srcLinks.put(oldLink.src().deviceId(), key);
+            dstLinks.put(oldLink.dst().deviceId(), key);
+            return new LinkEvent(LINK_UPDATED, newLink);
         }
         return null;
     }
 
     @Override
     public LinkEvent removeLink(ConnectPoint src, ConnectPoint dst) {
-        synchronized (this) {
-            Link link = links.remove(new LinkKey(src, dst));
+        final LinkKey key = new LinkKey(src, dst);
+        ConcurrentMap<ProviderId, LinkDescription> descs = getLinkDescriptions(key);
+        synchronized (descs) {
+            Link link = links.remove(key);
+            descs.clear();
             if (link != null) {
-                srcLinks.remove(link.src().deviceId(), link);
-                dstLinks.remove(link.dst().deviceId(), link);
+                srcLinks.remove(link.src().deviceId(), key);
+                dstLinks.remove(link.dst().deviceId(), key);
                 return new LinkEvent(LINK_REMOVED, link);
             }
             return null;
         }
     }
+
+    private static <K, V> SetMultimap<K, V> createSynchronizedHashMultiMap() {
+        return synchronizedSetMultimap(HashMultimap.<K, V>create());
+    }
+
+    /**
+     * @return primary ProviderID, or randomly chosen one if none exists
+     */
+    private ProviderId pickPrimaryPID(
+            ConcurrentMap<ProviderId, LinkDescription> providerDescs) {
+
+        ProviderId fallBackPrimary = null;
+        for (Entry<ProviderId, LinkDescription> e : providerDescs.entrySet()) {
+            if (!e.getKey().isAncillary()) {
+                return e.getKey();
+            } else if (fallBackPrimary == null) {
+                // pick randomly as a fallback in case there is no primary
+                fallBackPrimary = e.getKey();
+            }
+        }
+        return fallBackPrimary;
+    }
+
+    private Link composeLink(ConcurrentMap<ProviderId, LinkDescription> descs) {
+        ProviderId primary = pickPrimaryPID(descs);
+        LinkDescription base = descs.get(primary);
+
+        ConnectPoint src = base.src();
+        ConnectPoint dst = base.dst();
+        Type type = base.type();
+        DefaultAnnotations annotations = DefaultAnnotations.builder().build();
+        annotations = merge(annotations, base.annotations());
+
+        for (Entry<ProviderId, LinkDescription> e : descs.entrySet()) {
+            if (primary.equals(e.getKey())) {
+                continue;
+            }
+
+            // TODO: should keep track of Description timestamp
+            // and only merge conflicting keys when timestamp is newer
+            // Currently assuming there will never be a key conflict between
+            // providers
+
+            // annotation merging. not so efficient, should revisit later
+            annotations = merge(annotations, e.getValue().annotations());
+        }
+
+        return new DefaultLink(primary , src, dst, type, annotations);
+    }
+
+    private ConcurrentMap<ProviderId, LinkDescription> getLinkDescriptions(LinkKey key) {
+        return ConcurrentUtils.createIfAbsentUnchecked(linkDescs, key,
+                NewConcurrentHashMap.<ProviderId, LinkDescription>ifNeeded());
+    }
+
+    private final Function<LinkKey, Link> lookupLink = new LookupLink();
+    private Function<LinkKey, Link> lookupLink() {
+        return lookupLink;
+    }
+
+    private final class LookupLink implements Function<LinkKey, Link> {
+        @Override
+        public Link apply(LinkKey input) {
+            return links.get(input);
+        }
+    }
+
+    private static final Predicate<Provided> IS_PRIMARY = new IsPrimary();
+    private static final Predicate<Provided> isPrimary() {
+        return IS_PRIMARY;
+    }
+
+    private static final class IsPrimary implements Predicate<Provided> {
+
+        @Override
+        public boolean apply(Provided input) {
+            return !input.providerId().isAncillary();
+        }
+    }
 }

@@ -124,7 +124,8 @@ public class SimpleTopologyStore
         // Promote the new topology to current and return a ready-to-send event.
         synchronized (this) {
             current = newTopology;
-            return new TopologyEvent(TopologyEvent.Type.TOPOLOGY_CHANGED, current);
+            return new TopologyEvent(TopologyEvent.Type.TOPOLOGY_CHANGED,
+                                     current, reasons);
         }
     }
 

@@ -103,17 +103,19 @@ public class SimpleDeviceStoreTest {
         simpleDeviceStore.deactivate();
     }
 
-    private void putDevice(DeviceId deviceId, String swVersion) {
+    private void putDevice(DeviceId deviceId, String swVersion,
+                           SparseAnnotations... annotations) {
         DeviceDescription description =
                 new DefaultDeviceDescription(deviceId.uri(), SWITCH, MFR,
-                        HW, swVersion, SN);
+                        HW, swVersion, SN, annotations);
         deviceStore.createOrUpdateDevice(PID, deviceId, description);
     }
 
-    private void putDeviceAncillary(DeviceId deviceId, String swVersion) {
+    private void putDeviceAncillary(DeviceId deviceId, String swVersion,
+                                    SparseAnnotations... annotations) {
         DeviceDescription description =
                 new DefaultDeviceDescription(deviceId.uri(), SWITCH, MFR,
-                        HW, swVersion, SN);
+                        HW, swVersion, SN, annotations);
         deviceStore.createOrUpdateDevice(PIDA, deviceId, description);
     }
 
@@ -126,6 +128,7 @@ public class SimpleDeviceStoreTest {
         assertEquals(SN, device.serialNumber());
     }
 
+    // TODO slice this out somewhere
     /**
      * Verifies that Annotations created by merging {@code annotations} is
      * equal to actual Annotations.
@@ -133,7 +136,7 @@ public class SimpleDeviceStoreTest {
      * @param actual Annotations to check
      * @param annotations
      */
-    private static void assertAnnotationsEquals(Annotations actual, SparseAnnotations... annotations) {
+    public static void assertAnnotationsEquals(Annotations actual, SparseAnnotations... annotations) {
         DefaultAnnotations expected = DefaultAnnotations.builder().build();
         for (SparseAnnotations a : annotations) {
             expected = DefaultAnnotations.merge(expected, a);
@@ -347,6 +350,7 @@ public class SimpleDeviceStoreTest {
         assertFalse("Port is disabled", event.port().isEnabled());
 
     }
+
     @Test
     public final void testUpdatePortStatusAncillary() {
         putDeviceAncillary(DID1, SW1);
@@ -435,16 +439,37 @@ public class SimpleDeviceStoreTest {
 
     @Test
     public final void testRemoveDevice() {
-        putDevice(DID1, SW1);
+        putDevice(DID1, SW1, A1);
+        List<PortDescription> pds = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, true, A2)
+                );
+        deviceStore.updatePorts(PID, DID1, pds);
         putDevice(DID2, SW1);
 
         assertEquals(2, deviceStore.getDeviceCount());
+        assertEquals(1, deviceStore.getPorts(DID1).size());
+        assertAnnotationsEquals(deviceStore.getDevice(DID1).annotations(), A1);
+        assertAnnotationsEquals(deviceStore.getPort(DID1, P1).annotations(), A2);
 
         DeviceEvent event = deviceStore.removeDevice(DID1);
         assertEquals(DEVICE_REMOVED, event.type());
         assertDevice(DID1, SW1, event.subject());
 
         assertEquals(1, deviceStore.getDeviceCount());
+        assertEquals(0, deviceStore.getPorts(DID1).size());
+
+        // putBack Device, Port w/o annotation
+        putDevice(DID1, SW1);
+        List<PortDescription> pds2 = Arrays.<PortDescription>asList(
+                new DefaultPortDescription(P1, true)
+                );
+        deviceStore.updatePorts(PID, DID1, pds2);
+
+        // annotations should not survive
+        assertEquals(2, deviceStore.getDeviceCount());
+        assertEquals(1, deviceStore.getPorts(DID1).size());
+        assertAnnotationsEquals(deviceStore.getDevice(DID1).annotations());
+        assertAnnotationsEquals(deviceStore.getPort(DID1, P1).annotations());
     }
 
     // If Delegates should be called only on remote events,

@@ -4,7 +4,9 @@ import static org.junit.Assert.*;
 import static org.onlab.onos.net.DeviceId.deviceId;
 import static org.onlab.onos.net.Link.Type.*;
 import static org.onlab.onos.net.link.LinkEvent.Type.*;
+import static org.onlab.onos.store.trivial.impl.SimpleDeviceStoreTest.assertAnnotationsEquals;
 
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.Map;
 import java.util.Set;
@@ -18,10 +20,12 @@ import org.junit.BeforeClass;
 import org.junit.Ignore;
 import org.junit.Test;
 import org.onlab.onos.net.ConnectPoint;
+import org.onlab.onos.net.DefaultAnnotations;
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.Link;
 import org.onlab.onos.net.LinkKey;
 import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.SparseAnnotations;
 import org.onlab.onos.net.Link.Type;
 import org.onlab.onos.net.link.DefaultLinkDescription;
 import org.onlab.onos.net.link.LinkEvent;
@@ -37,6 +41,7 @@ import com.google.common.collect.Iterables;
 public class SimpleLinkStoreTest {
 
     private static final ProviderId PID = new ProviderId("of", "foo");
+    private static final ProviderId PIDA = new ProviderId("of", "bar", true);
     private static final DeviceId DID1 = deviceId("of:foo");
     private static final DeviceId DID2 = deviceId("of:bar");
 
@@ -44,6 +49,23 @@ public class SimpleLinkStoreTest {
     private static final PortNumber P2 = PortNumber.portNumber(2);
     private static final PortNumber P3 = PortNumber.portNumber(3);
 
+    private static final SparseAnnotations A1 = DefaultAnnotations.builder()
+            .set("A1", "a1")
+            .set("B1", "b1")
+            .build();
+    private static final SparseAnnotations A1_2 = DefaultAnnotations.builder()
+            .remove("A1")
+            .set("B3", "b3")
+            .build();
+    private static final SparseAnnotations A2 = DefaultAnnotations.builder()
+            .set("A2", "a2")
+            .set("B2", "b2")
+            .build();
+    private static final SparseAnnotations A2_2 = DefaultAnnotations.builder()
+            .remove("A2")
+            .set("B4", "b4")
+            .build();
+
 
     private SimpleLinkStore simpleLinkStore;
     private LinkStore linkStore;
@@ -69,16 +91,17 @@ public class SimpleLinkStoreTest {
     }
 
     private void putLink(DeviceId srcId, PortNumber srcNum,
-                         DeviceId dstId, PortNumber dstNum, Type type) {
+                         DeviceId dstId, PortNumber dstNum, Type type,
+                         SparseAnnotations... annotations) {
         ConnectPoint src = new ConnectPoint(srcId, srcNum);
         ConnectPoint dst = new ConnectPoint(dstId, dstNum);
-        linkStore.createOrUpdateLink(PID, new DefaultLinkDescription(src, dst, type));
+        linkStore.createOrUpdateLink(PID, new DefaultLinkDescription(src, dst, type, annotations));
     }
 
-    private void putLink(LinkKey key, Type type) {
+    private void putLink(LinkKey key, Type type, SparseAnnotations... annotations) {
         putLink(key.src().deviceId(), key.src().port(),
                 key.dst().deviceId(), key.dst().port(),
-                type);
+                type, annotations);
     }
 
     private static void assertLink(DeviceId srcId, PortNumber srcNum,
@@ -270,14 +293,67 @@ public class SimpleLinkStoreTest {
     }
 
     @Test
+    public final void testCreateOrUpdateLinkAncillary() {
+        ConnectPoint src = new ConnectPoint(DID1, P1);
+        ConnectPoint dst = new ConnectPoint(DID2, P2);
+
+        // add Ancillary link
+        LinkEvent event = linkStore.createOrUpdateLink(PIDA,
+                    new DefaultLinkDescription(src, dst, INDIRECT, A1));
+
+        assertNull("Ancillary only link is ignored", event);
+
+        // add Primary link
+        LinkEvent event2 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, INDIRECT, A2));
+
+        assertLink(DID1, P1, DID2, P2, INDIRECT, event2.subject());
+        assertAnnotationsEquals(event2.subject().annotations(), A2, A1);
+        assertEquals(LINK_ADDED, event2.type());
+
+        // update link type
+        LinkEvent event3 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT, A2));
+        assertLink(DID1, P1, DID2, P2, DIRECT, event3.subject());
+        assertAnnotationsEquals(event3.subject().annotations(), A2, A1);
+        assertEquals(LINK_UPDATED, event3.type());
+
+
+        // no change
+        LinkEvent event4 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT));
+        assertNull("No change event expected", event4);
+
+        // update link annotation (Primary)
+        LinkEvent event5 = linkStore.createOrUpdateLink(PID,
+                new DefaultLinkDescription(src, dst, DIRECT, A2_2));
+        assertLink(DID1, P1, DID2, P2, DIRECT, event5.subject());
+        assertAnnotationsEquals(event5.subject().annotations(), A2, A2_2, A1);
+        assertEquals(LINK_UPDATED, event5.type());
+
+        // update link annotation (Ancillary)
+        LinkEvent event6 = linkStore.createOrUpdateLink(PIDA,
+                new DefaultLinkDescription(src, dst, DIRECT, A1_2));
+        assertLink(DID1, P1, DID2, P2, DIRECT, event6.subject());
+        assertAnnotationsEquals(event6.subject().annotations(), A2, A2_2, A1, A1_2);
+        assertEquals(LINK_UPDATED, event6.type());
+
+        // update link type (Ancillary) : ignored
+        LinkEvent event7 = linkStore.createOrUpdateLink(PIDA,
+                new DefaultLinkDescription(src, dst, EDGE));
+        assertNull("Ancillary change other than annotation is ignored", event7);
+    }
+
+
+    @Test
     public final void testRemoveLink() {
         final ConnectPoint d1P1 = new ConnectPoint(DID1, P1);
         final ConnectPoint d2P2 = new ConnectPoint(DID2, P2);
         LinkKey linkId1 = new LinkKey(d1P1, d2P2);
         LinkKey linkId2 = new LinkKey(d2P2, d1P1);
 
-        putLink(linkId1, DIRECT);
-        putLink(linkId2, DIRECT);
+        putLink(linkId1, DIRECT, A1);
+        putLink(linkId2, DIRECT, A2);
 
         // DID1,P1 => DID2,P2
         // DID2,P2 => DID1,P1
@@ -285,10 +361,41 @@ public class SimpleLinkStoreTest {
 
         LinkEvent event = linkStore.removeLink(d1P1, d2P2);
         assertEquals(LINK_REMOVED, event.type());
+        assertAnnotationsEquals(event.subject().annotations(), A1);
         LinkEvent event2 = linkStore.removeLink(d1P1, d2P2);
         assertNull(event2);
 
         assertLink(linkId2, DIRECT, linkStore.getLink(d2P2, d1P1));
+        assertAnnotationsEquals(linkStore.getLink(d2P2, d1P1).annotations(), A2);
+
+        // annotations, etc. should not survive remove
+        putLink(linkId1, DIRECT);
+        assertLink(linkId1, DIRECT, linkStore.getLink(d1P1, d2P2));
+        assertAnnotationsEquals(linkStore.getLink(d1P1, d2P2).annotations());
+    }
+
+    @Test
+    public final void testAncillaryOnlyNotVisible() {
+        ConnectPoint src = new ConnectPoint(DID1, P1);
+        ConnectPoint dst = new ConnectPoint(DID2, P2);
+
+        // add Ancillary link
+        linkStore.createOrUpdateLink(PIDA,
+                    new DefaultLinkDescription(src, dst, INDIRECT, A1));
+
+        // Ancillary only link should not be visible
+        assertEquals(0, linkStore.getLinkCount());
+
+        assertTrue(Iterables.isEmpty(linkStore.getLinks()));
+
+        assertNull(linkStore.getLink(src, dst));
+
+        assertEquals(Collections.emptySet(), linkStore.getIngressLinks(dst));
+
+        assertEquals(Collections.emptySet(), linkStore.getEgressLinks(src));
+
+        assertEquals(Collections.emptySet(), linkStore.getDeviceEgressLinks(DID1));
+        assertEquals(Collections.emptySet(), linkStore.getDeviceIngressLinks(DID2));
     }
 
     // If Delegates should be called only on remote events,

@@ -9,6 +9,12 @@
         <bundle>mvn:org.apache.commons/commons-lang3/3.3.2</bundle>
         <bundle>mvn:com.google.guava/guava/18.0</bundle>
         <bundle>mvn:io.netty/netty/3.9.2.Final</bundle>
+        <bundle>mvn:io.netty/netty-common/4.0.23.Final</bundle>
+        <bundle>mvn:io.netty/netty-buffer/4.0.23.Final</bundle>
+        <bundle>mvn:io.netty/netty-transport/4.0.23.Final</bundle>
+        <bundle>mvn:io.netty/netty-handler/4.0.23.Final</bundle>
+        <bundle>mvn:io.netty/netty-codec/4.0.23.Final</bundle>
+        <bundle>mvn:commons-pool/commons-pool/1.6</bundle>
 
         <bundle>mvn:com.hazelcast/hazelcast/3.3</bundle>
         <bundle>mvn:io.dropwizard.metrics/metrics-core/3.1.0</bundle>
@@ -53,6 +59,11 @@
         <feature>onos-api</feature>
         <bundle>mvn:org.onlab.onos/onos-core-net/1.0.0-SNAPSHOT</bundle>
         <bundle>mvn:org.onlab.onos/onos-core-dist/1.0.0-SNAPSHOT</bundle>
+        <bundle>mvn:org.onlab.onos/onos-core-serializers/1.0.0-SNAPSHOT</bundle>
+        <bundle>mvn:org.onlab.onos/onlab-netty/1.0.0-SNAPSHOT</bundle>
+
+        <bundle>mvn:org.onlab.onos/onos-core-hz-common/1.0.0-SNAPSHOT</bundle>
+        <bundle>mvn:org.onlab.onos/onos-core-hz-cluster/1.0.0-SNAPSHOT</bundle>
     </feature>
 
     <feature name="onos-core-hazelcast" version="1.0.0"
@@ -120,18 +131,29 @@
         <bundle>mvn:org.onlab.onos/onos-app-fwd/1.0.0-SNAPSHOT</bundle>
     </feature>
 
+    <feature name="onos-app-ifwd" version="1.0.0"
+             description="ONOS sample forwarding application using intents">
+        <feature>onos-api</feature>
+        <bundle>mvn:org.onlab.onos/onos-app-ifwd/1.0.0-SNAPSHOT</bundle>
+    </feature>
+
     <feature name="onos-app-mobility" version="1.0.0"
-             description="ONOS sample forwarding application">
+             description="ONOS sample mobility application">
         <feature>onos-api</feature>
         <bundle>mvn:org.onlab.onos/onos-app-mobility/1.0.0-SNAPSHOT</bundle>
     </feature>
 
-
+    <feature name="onos-app-proxyarp" version="1.0.0"
+             description="ONOS sample proxyarp application">
+        <feature>onos-api</feature>
+        <bundle>mvn:org.onlab.onos/onos-app-proxyarp/1.0.0-SNAPSHOT</bundle>
+    </feature>
 
     <feature name="onos-app-foo" version="1.0.0"
              description="ONOS sample playground application">
         <feature>onos-api</feature>
         <bundle>mvn:org.onlab.onos/onos-app-foo/1.0.0-SNAPSHOT</bundle>
+        <bundle>mvn:org.onlab.onos/onlab-netty/1.0.0-SNAPSHOT</bundle>
     </feature>
 
     <feature name="onos-app-config" version="1.0.0"

-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<!--
-  ~ Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
-  ~
-  ~ This program and the accompanying materials are made available under the
-  ~ terms of the Eclipse Public License v1.0 which accompanies this distribution,
-  ~ and is available at http://www.eclipse.org/legal/epl-v10.html
-  -->
-
-<features xmlns="http://karaf.apache.org/xmlns/features/v1.2.0"
-          name="net.onrc.onos-1.0.0">
-    <repository>mvn:net.onrc.onos/onos-features/1.0.0-SNAPSHOT/xml/features</repository>
-
-    <feature name="thirdparty" version="1.0.0"
-             description="ONOS 3rd party dependencies">
-        <bundle>mvn:com.google.code.findbugs/annotations/2.0.2</bundle>
-        <bundle>mvn:io.netty/netty/3.9.2.Final</bundle>
-        <bundle>mvn:com.google.guava/guava/17.0</bundle>  
-        <bundle>mvn:com.google.guava/guava/15.0</bundle>  
-
-    </feature>
-
-    <feature name="base" version="1.0.0"
-            description="ONOS Base">
-        <feature>scr</feature>
-        <feature>thirdparty</feature>
-        <bundle>mvn:net.onrc.onos.sb/onos-sb/0.0.1</bundle>
-        <bundle>mvn:org.projectfloodlight/openflowj/0.3.6-SNAPSHOT</bundle>
-    </feature>
-
-</features>

@@ -17,14 +17,11 @@
     <description>ONOS OpenFlow controller subsystem API</description>
 
     <repositories>
-        <!-- FIXME: for Loxigen. Decide how to use Loxigen before release. -->
+        <!-- FIXME: for Loxigen + optical experimenter. Decide how to use Loxigen before release. -->
         <repository>
-            <id>sonatype-oss-snapshot</id>
-            <name>Sonatype OSS snapshot repository</name>
-            <url>https://oss.sonatype.org/content/repositories/snapshots</url>
-            <releases>
-                <enabled>false</enabled>
-            </releases>
+            <id>onlab-temp</id>
+            <name>ON.lab temporary repository</name>
+            <url>http://mavenrepo.onlab.us:8081/nexus/content/repositories/releases</url>
         </repository>
     </repositories>
 
@@ -32,7 +29,8 @@
         <dependency>
             <groupId>org.projectfloodlight</groupId>
             <artifactId>openflowj</artifactId>
-            <version>0.3.8-SNAPSHOT</version>
+            <!-- FIXME once experimenter gets merged to upstream -->
+            <version>0.3.8-optical_experimenter</version>
         </dependency>
         <dependency>
             <groupId>io.netty</groupId>

@@ -167,11 +167,11 @@ class OFChannelHandler extends IdleStateAwareChannelHandler {
                 // TODO We could check for the optional bitmap, but for now
                 // we are just checking the version number.
                 if (m.getVersion() == OFVersion.OF_13) {
-                    log.info("Received {} Hello from {}", m.getVersion(),
+                    log.debug("Received {} Hello from {}", m.getVersion(),
                             h.channel.getRemoteAddress());
                     h.ofVersion = OFVersion.OF_13;
                 } else if (m.getVersion() == OFVersion.OF_10) {
-                    log.info("Received {} Hello from {} - switching to OF "
+                    log.debug("Received {} Hello from {} - switching to OF "
                             + "version 1.0", m.getVersion(),
                             h.channel.getRemoteAddress());
                     h.ofVersion = OFVersion.OF_10;
@@ -222,7 +222,7 @@ class OFChannelHandler extends IdleStateAwareChannelHandler {
             void processOFFeaturesReply(OFChannelHandler h, OFFeaturesReply  m)
                     throws IOException {
                 h.thisdpid = m.getDatapathId().getLong();
-                log.info("Received features reply for switch at {} with dpid {}",
+                log.debug("Received features reply for switch at {} with dpid {}",
                         h.getSwitchInfoString(), h.thisdpid);
 
                 h.featuresReply = m; //temp store
@@ -409,7 +409,7 @@ class OFChannelHandler extends IdleStateAwareChannelHandler {
 
 
 
-                log.info("Switch {} bound to class {}, description {}",
+                log.debug("Switch {} bound to class {}, description {}",
                         new Object[] {h.sw, h.sw.getClass(), drep });
                 //Put switch in EQUAL mode until we hear back from the global registry
                 //log.debug("Setting new switch {} to EQUAL and sending Role request",
@@ -651,7 +651,7 @@ class OFChannelHandler extends IdleStateAwareChannelHandler {
          * @param error The error message
          */
         protected void logError(OFChannelHandler h, OFErrorMsg error) {
-            log.info("{} from switch {} in state {}",
+            log.error("{} from switch {} in state {}",
                     new Object[] {
                     error,
                     h.getSwitchInfoString(),
@@ -1052,7 +1052,7 @@ class OFChannelHandler extends IdleStateAwareChannelHandler {
             throws Exception {
         OFFactory factory = (ofVersion == OFVersion.OF_13) ? factory13 : factory10;
         OFMessage m = factory.buildEchoRequest().build();
-        log.info("Sending Echo Request on idle channel: {}",
+        log.debug("Sending Echo Request on idle channel: {}",
                 e.getChannel().getPipeline().getLast().toString());
         e.getChannel().write(Collections.singletonList(m));
         // XXX S some problems here -- echo request has no transaction id, and

@@ -122,7 +122,7 @@ class RoleManager implements RoleHandler {
                 //FIXME fix below when we actually use generation ids
                 .setGenerationId(U64.ZERO)
                 .build();
-        sw.sendMsg(rrm);
+        sw.write(rrm);
         return xid;
     }
 

@@ -138,7 +138,7 @@ public class OFSwitchImplOVS13 extends AbstractOpenFlowSwitch {
                 .buildBarrierRequest()
                 .setXid(xid)
                 .build();
-        sendMsg(br);
+        write(br);
     }
 
     @Override
@@ -227,7 +227,7 @@ public class OFSwitchImplOVS13 extends AbstractOpenFlowSwitch {
                 .setHardTimeout(0)
                 .setXid(getNextTransactionId())
                 .build();
-        sendMsg(tableMissEntry);
+        write(tableMissEntry);
     }
 
 }

@@ -40,15 +40,15 @@ public enum OFType {
      PORT_STATUS,
      PACKET_OUT,
      FLOW_MOD,
+     GROUP_MOD,
      PORT_MOD,
+     TABLE_MOD,
      STATS_REQUEST,
      STATS_REPLY,
      BARRIER_REQUEST,
      BARRIER_REPLY,
      QUEUE_GET_CONFIG_REQUEST,
      QUEUE_GET_CONFIG_REPLY,
-     GROUP_MOD,
-     TABLE_MOD,
      ROLE_REQUEST,
      ROLE_REPLY,
      GET_ASYNC_REQUEST,

@@ -69,6 +69,13 @@
 
             <dependency>
                 <groupId>org.slf4j</groupId>
+                <artifactId>slf4j-core</artifactId>
+                <version>1.7.6</version>
+                <scope>test</scope>
+            </dependency>
+
+            <dependency>
+                <groupId>org.slf4j</groupId>
                 <artifactId>slf4j-jdk14</artifactId>
                 <version>1.7.6</version>
                 <scope>test</scope>
@@ -122,6 +129,12 @@
                 <version>1.9.13</version>
             </dependency>
 
+	    <dependency>
+	        <groupId>org.easymock</groupId>
+	        <artifactId>easymock</artifactId>
+	        <version>3.2</version>
+	        <scope>test</scope>
+	    </dependency>
 
             <!-- Web related -->
             <dependency>
@@ -480,7 +493,7 @@
                         <group>
                             <title>Core Subsystems</title>
                             <packages>
-                                org.onlab.onos.cluster.impl:org.onlab.onos.net.device.impl:org.onlab.onos.net.link.impl:org.onlab.onos.net.host.impl:org.onlab.onos.net.topology.impl:org.onlab.onos.net.packet.impl:org.onlab.onos.net.flow.impl:org.onlab.onos.store.trivial.*:org.onlab.onos.net.*.impl:org.onlab.onos.event.impl:org.onlab.onos.store.*
+                                org.onlab.onos.cluster.impl:org.onlab.onos.net.device.impl:org.onlab.onos.net.link.impl:org.onlab.onos.net.host.impl:org.onlab.onos.net.topology.impl:org.onlab.onos.net.packet.impl:org.onlab.onos.net.flow.impl:org.onlab.onos.store.trivial.*:org.onlab.onos.net.*.impl:org.onlab.onos.event.impl:org.onlab.onos.store.*:org.onlab.onos.net.intent.impl
                             </packages>
                         </group>
                         <group>

@@ -6,11 +6,13 @@ import java.util.List;
 
 import org.onlab.onos.net.DeviceId;
 import org.onlab.onos.net.PortNumber;
+import org.onlab.onos.net.flow.DefaultFlowEntry;
 import org.onlab.onos.net.flow.DefaultFlowRule;
 import org.onlab.onos.net.flow.DefaultTrafficSelector;
 import org.onlab.onos.net.flow.DefaultTrafficTreatment;
+import org.onlab.onos.net.flow.FlowEntry;
+import org.onlab.onos.net.flow.FlowEntry.FlowEntryState;
 import org.onlab.onos.net.flow.FlowRule;
-import org.onlab.onos.net.flow.FlowRule.FlowRuleState;
 import org.onlab.onos.net.flow.TrafficSelector;
 import org.onlab.onos.net.flow.TrafficTreatment;
 import org.onlab.onos.openflow.controller.Dpid;
@@ -18,8 +20,8 @@ import org.onlab.packet.IpPrefix;
 import org.onlab.packet.MacAddress;
 import org.onlab.packet.VlanId;
 import org.projectfloodlight.openflow.protocol.OFFlowRemoved;
-import org.projectfloodlight.openflow.protocol.OFFlowRemovedReason;
 import org.projectfloodlight.openflow.protocol.OFFlowStatsEntry;
+import org.projectfloodlight.openflow.protocol.OFInstructionType;
 import org.projectfloodlight.openflow.protocol.action.OFAction;
 import org.projectfloodlight.openflow.protocol.action.OFActionOutput;
 import org.projectfloodlight.openflow.protocol.action.OFActionSetDlDst;
@@ -28,12 +30,16 @@ import org.projectfloodlight.openflow.protocol.action.OFActionSetNwDst;
 import org.projectfloodlight.openflow.protocol.action.OFActionSetNwSrc;
 import org.projectfloodlight.openflow.protocol.action.OFActionSetVlanPcp;
 import org.projectfloodlight.openflow.protocol.action.OFActionSetVlanVid;
+import org.projectfloodlight.openflow.protocol.instruction.OFInstruction;
+import org.projectfloodlight.openflow.protocol.instruction.OFInstructionApplyActions;
 import org.projectfloodlight.openflow.protocol.match.Match;
 import org.projectfloodlight.openflow.protocol.match.MatchField;
 import org.projectfloodlight.openflow.types.IPv4Address;
 import org.slf4j.Logger;
 
-public class FlowRuleBuilder {
+import com.google.common.collect.Lists;
+
+public class FlowEntryBuilder {
     private final Logger log = getLogger(getClass());
 
     private final OFFlowStatsEntry stat;
@@ -44,49 +50,70 @@ public class FlowRuleBuilder {
 
     private final Dpid dpid;
 
+    private final boolean addedRule;
 
 
-
-    public FlowRuleBuilder(Dpid dpid, OFFlowStatsEntry entry) {
+    public FlowEntryBuilder(Dpid dpid, OFFlowStatsEntry entry) {
         this.stat = entry;
         this.match = entry.getMatch();
-        this.actions = entry.getActions();
+        this.actions = getActions(entry);
         this.dpid = dpid;
         this.removed = null;
+        this.addedRule = true;
     }
 
-    public FlowRuleBuilder(Dpid dpid, OFFlowRemoved removed) {
+    public FlowEntryBuilder(Dpid dpid, OFFlowRemoved removed) {
         this.match = removed.getMatch();
         this.removed = removed;
 
         this.dpid = dpid;
         this.actions = null;
         this.stat = null;
+        this.addedRule = false;
 
     }
 
-    public FlowRule build() {
-        if (stat != null) {
-            return new DefaultFlowRule(DeviceId.deviceId(Dpid.uri(dpid)),
+    public FlowEntry build() {
+        if (addedRule) {
+            FlowRule rule = new DefaultFlowRule(DeviceId.deviceId(Dpid.uri(dpid)),
                     buildSelector(), buildTreatment(), stat.getPriority(),
-                    FlowRuleState.ADDED, stat.getDurationNsec() / 1000000,
-                    stat.getPacketCount().getValue(), stat.getByteCount().getValue(),
-                    stat.getCookie().getValue(), false, stat.getIdleTimeout());
+                    stat.getCookie().getValue(), stat.getIdleTimeout());
+            return new DefaultFlowEntry(rule, FlowEntryState.ADDED,
+                    stat.getDurationSec(), stat.getPacketCount().getValue(),
+                    stat.getByteCount().getValue());
+
         } else {
-            // TODO: revisit potentially.
-            return new DefaultFlowRule(DeviceId.deviceId(Dpid.uri(dpid)),
+            FlowRule rule = new DefaultFlowRule(DeviceId.deviceId(Dpid.uri(dpid)),
                     buildSelector(), null, removed.getPriority(),
-                    FlowRuleState.REMOVED, removed.getDurationNsec() / 1000000,
-                    removed.getPacketCount().getValue(), removed.getByteCount().getValue(),
-                    removed.getCookie().getValue(),
-                    removed.getReason() == OFFlowRemovedReason.IDLE_TIMEOUT.ordinal(),
-                    stat.getIdleTimeout());
+                   removed.getCookie().getValue(), removed.getIdleTimeout());
+            return new DefaultFlowEntry(rule, FlowEntryState.REMOVED, removed.getDurationSec(),
+                    removed.getPacketCount().getValue(), removed.getByteCount().getValue());
         }
     }
 
+    private List<OFAction> getActions(OFFlowStatsEntry entry) {
+        switch (entry.getVersion()) {
+            case OF_10:
+                return entry.getActions();
+            case OF_11:
+            case OF_12:
+            case OF_13:
+                List<OFInstruction> ins = entry.getInstructions();
+                for (OFInstruction in : ins) {
+                    if (in.getType().equals(OFInstructionType.APPLY_ACTIONS)) {
+                        OFInstructionApplyActions apply = (OFInstructionApplyActions) in;
+                        return apply.getActions();
+                    }
+                }
+                return Lists.newLinkedList();
+            default:
+                log.warn("Unknown OF version {}", entry.getVersion());
+        }
+        return Lists.newLinkedList();
+    }
 
     private TrafficTreatment buildTreatment() {
-        TrafficTreatment.Builder builder = new DefaultTrafficTreatment.Builder();
+        TrafficTreatment.Builder builder = DefaultTrafficTreatment.builder();
         // If this is a drop rule
         if (actions.size() == 0) {
             builder.drop();
@@ -171,7 +198,7 @@ public class FlowRuleBuilder {
     }
 
     private TrafficSelector buildSelector() {
-        TrafficSelector.Builder builder = new DefaultTrafficSelector.Builder();
+        TrafficSelector.Builder builder = DefaultTrafficSelector.builder();
         for (MatchField<?> field : match.getMatchFields()) {
             switch (field.id) {
             case IN_PORT:

@@ -68,7 +68,7 @@ public class FlowModBuilder {
         this.cookie = flowRule.id();
     }
 
-    public OFFlowMod buildFlowMod() {
+    public OFFlowMod buildFlowAdd() {
         Match match = buildMatch();
         List<OFAction> actions = buildActions();
 
@@ -86,6 +86,24 @@ public class FlowModBuilder {
 
     }
 
+    public OFFlowMod buildFlowMod() {
+        Match match = buildMatch();
+        List<OFAction> actions = buildActions();
+
+        //TODO: what to do without bufferid? do we assume that there will be a pktout as well?
+        OFFlowMod fm = factory.buildFlowModify()
+                .setCookie(U64.of(cookie.value()))
+                .setBufferId(OFBufferId.NO_BUFFER)
+                .setActions(actions)
+                .setMatch(match)
+                .setFlags(Collections.singleton(OFFlowModFlags.SEND_FLOW_REM))
+                .setPriority(priority)
+                .build();
+
+        return fm;
+
+    }
+
     public OFFlowMod buildFlowDel() {
         Match match = buildMatch();
         List<OFAction> actions = buildActions();

@@ -2,7 +2,17 @@ package org.onlab.onos.provider.of.flow.impl;
 
 import static org.slf4j.LoggerFactory.getLogger;
 
+import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
 
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
@@ -11,10 +21,13 @@ import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.onlab.onos.ApplicationId;
 import org.onlab.onos.net.DeviceId;
+import org.onlab.onos.net.flow.FlowEntry;
 import org.onlab.onos.net.flow.FlowRule;
+import org.onlab.onos.net.flow.FlowRuleBatchEntry;
 import org.onlab.onos.net.flow.FlowRuleProvider;
 import org.onlab.onos.net.flow.FlowRuleProviderRegistry;
 import org.onlab.onos.net.flow.FlowRuleProviderService;
+import org.onlab.onos.net.intent.BatchOperation;
 import org.onlab.onos.net.provider.AbstractProvider;
 import org.onlab.onos.net.provider.ProviderId;
 import org.onlab.onos.net.topology.TopologyService;
@@ -24,17 +37,29 @@ import org.onlab.onos.openflow.controller.OpenFlowEventListener;
 import org.onlab.onos.openflow.controller.OpenFlowSwitch;
 import org.onlab.onos.openflow.controller.OpenFlowSwitchListener;
 import org.onlab.onos.openflow.controller.RoleState;
+import org.projectfloodlight.openflow.protocol.OFActionType;
+import org.projectfloodlight.openflow.protocol.OFBarrierRequest;
+import org.projectfloodlight.openflow.protocol.OFErrorMsg;
 import org.projectfloodlight.openflow.protocol.OFFlowRemoved;
 import org.projectfloodlight.openflow.protocol.OFFlowStatsEntry;
 import org.projectfloodlight.openflow.protocol.OFFlowStatsReply;
+import org.projectfloodlight.openflow.protocol.OFInstructionType;
 import org.projectfloodlight.openflow.protocol.OFMessage;
 import org.projectfloodlight.openflow.protocol.OFPortStatus;
 import org.projectfloodlight.openflow.protocol.OFStatsReply;
 import org.projectfloodlight.openflow.protocol.OFStatsReplyFlags;
 import org.projectfloodlight.openflow.protocol.OFStatsType;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+import org.projectfloodlight.openflow.protocol.action.OFActionOutput;
+import org.projectfloodlight.openflow.protocol.instruction.OFInstruction;
+import org.projectfloodlight.openflow.protocol.instruction.OFInstructionApplyActions;
+import org.projectfloodlight.openflow.types.OFPort;
+import org.projectfloodlight.openflow.types.U32;
 import org.slf4j.Logger;
 
 import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Lists;
 import com.google.common.collect.Maps;
 import com.google.common.collect.Multimap;
 
@@ -60,6 +85,9 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
 
     private final InternalFlowProvider listener = new InternalFlowProvider();
 
+    private final Map<Long, InstallationFuture> pendingFutures =
+            new ConcurrentHashMap<Long, InstallationFuture>();
+
     /**
      * Creates an OpenFlow host provider.
      */
@@ -91,7 +119,7 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
 
     private void applyRule(FlowRule flowRule) {
         OpenFlowSwitch sw = controller.getSwitch(Dpid.dpid(flowRule.deviceId().uri()));
-        sw.sendMsg(new FlowModBuilder(flowRule, sw.factory()).buildFlowMod());
+        sw.sendMsg(new FlowModBuilder(flowRule, sw.factory()).buildFlowAdd());
     }
 
 
@@ -122,7 +150,7 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
     implements OpenFlowSwitchListener, OpenFlowEventListener {
 
         private final Map<Dpid, FlowStatsCollector> collectors = Maps.newHashMap();
-        private final Multimap<DeviceId, FlowRule> completeEntries =
+        private final Multimap<DeviceId, FlowEntry> completeEntries =
                 ArrayListMultimap.create();
 
         @Override
@@ -144,19 +172,30 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
 
         @Override
         public void handleMessage(Dpid dpid, OFMessage msg) {
+            InstallationFuture future = null;
             switch (msg.getType()) {
             case FLOW_REMOVED:
                 //TODO: make this better
                 OFFlowRemoved removed = (OFFlowRemoved) msg;
 
-                FlowRule fr = new FlowRuleBuilder(dpid, removed).build();
+                FlowEntry fr = new FlowEntryBuilder(dpid, removed).build();
                 providerService.flowRemoved(fr);
                 break;
             case STATS_REPLY:
                 pushFlowMetrics(dpid, (OFStatsReply) msg);
                 break;
             case BARRIER_REPLY:
+                future = pendingFutures.get(msg.getXid());
+                if (future != null) {
+                    future.satisfyRequirement(dpid);
+                }
+                break;
             case ERROR:
+                future = pendingFutures.get(msg.getXid());
+                if (future != null) {
+                    future.fail((OFErrorMsg) msg, dpid);
+                }
+                break;
             default:
                 log.debug("Unhandled message type: {}", msg.getType());
             }
@@ -178,8 +217,9 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
             //final List<FlowRule> entries = Lists.newLinkedList();
 
             for (OFFlowStatsEntry reply : replies.getEntries()) {
-                completeEntries.put(did, new FlowRuleBuilder(dpid, reply).build());
-                //entries.add(new FlowRuleBuilder(dpid, reply).build());
+                if (!tableMissRule(dpid, reply)) {
+                    completeEntries.put(did, new FlowEntryBuilder(dpid, reply).build());
+                }
             }
 
             if (!stats.getFlags().contains(OFStatsReplyFlags.REPLY_MORE)) {
@@ -189,9 +229,170 @@ public class OpenFlowRuleProvider extends AbstractProvider implements FlowRulePr
             }
         }
 
+        private boolean tableMissRule(Dpid dpid, OFFlowStatsEntry reply) {
+            // TODO NEED TO FIND A BETTER WAY TO AVOID DOING THIS
+            if (reply.getVersion().equals(OFVersion.OF_10) ||
+                    reply.getMatch().getMatchFields().iterator().hasNext()) {
+                return false;
+            }
+            for (OFInstruction ins : reply.getInstructions()) {
+                if (ins.getType() == OFInstructionType.APPLY_ACTIONS) {
+                    OFInstructionApplyActions apply = (OFInstructionApplyActions) ins;
+                    List<OFAction> acts = apply.getActions();
+                    for (OFAction act : acts) {
+                        if (act.getType() == OFActionType.OUTPUT) {
+                            OFActionOutput out = (OFActionOutput) act;
+                            if (out.getPort() == OFPort.CONTROLLER) {
+                                return true;
+                            }
+                        }
+                    }
+                }
+            }
+            return false;
+        }
+
     }
 
 
+    @Override
+    public Future<Void> executeBatch(BatchOperation<FlowRuleBatchEntry> batch) {
+        final Set<Dpid> sws = new HashSet<Dpid>();
+
+        for (FlowRuleBatchEntry fbe : batch.getOperations()) {
+            FlowRule flowRule = fbe.getTarget();
+            OpenFlowSwitch sw = controller.getSwitch(Dpid.dpid(flowRule.deviceId().uri()));
+            sws.add(new Dpid(sw.getId()));
+            switch (fbe.getOperator()) {
+                case ADD:
+                  //TODO: Track XID for each flowmod
+                    sw.sendMsg(new FlowModBuilder(flowRule, sw.factory()).buildFlowAdd());
+                    break;
+                case REMOVE:
+                  //TODO: Track XID for each flowmod
+                    sw.sendMsg(new FlowModBuilder(flowRule, sw.factory()).buildFlowDel());
+                    break;
+                case MODIFY:
+                  //TODO: Track XID for each flowmod
+                    sw.sendMsg(new FlowModBuilder(flowRule, sw.factory()).buildFlowMod());
+                    break;
+                default:
+                    log.error("Unsupported batch operation {}", fbe.getOperator());
+            }
+        }
+        InstallationFuture installation = new InstallationFuture(sws);
+        pendingFutures.put(U32.f(batch.hashCode()), installation);
+        installation.verify(batch.hashCode());
+        return installation;
+    }
+
+    private class InstallationFuture implements Future<Void> {
+
+        private final Set<Dpid> sws;
+        private final AtomicBoolean ok = new AtomicBoolean(true);
+        private final List<FlowEntry> offendingFlowMods = Lists.newLinkedList();
+
+        private final CountDownLatch countDownLatch;
+
+        public InstallationFuture(Set<Dpid> sws) {
+            this.sws = sws;
+            countDownLatch = new CountDownLatch(sws.size());
+        }
+
+        public void fail(OFErrorMsg msg, Dpid dpid) {
+            ok.set(false);
+            //TODO add reason to flowentry
+            //TODO handle specific error msgs
+            //offendingFlowMods.add(new FlowEntryBuilder(dpid, msg.));
+            switch (msg.getErrType()) {
+                case BAD_ACTION:
+                    break;
+                case BAD_INSTRUCTION:
+                    break;
+                case BAD_MATCH:
+                    break;
+                case BAD_REQUEST:
+                    break;
+                case EXPERIMENTER:
+                    break;
+                case FLOW_MOD_FAILED:
+                    break;
+                case GROUP_MOD_FAILED:
+                    break;
+                case HELLO_FAILED:
+                    break;
+                case METER_MOD_FAILED:
+                    break;
+                case PORT_MOD_FAILED:
+                    break;
+                case QUEUE_OP_FAILED:
+                    break;
+                case ROLE_REQUEST_FAILED:
+                    break;
+                case SWITCH_CONFIG_FAILED:
+                    break;
+                case TABLE_FEATURES_FAILED:
+                    break;
+                case TABLE_MOD_FAILED:
+                    break;
+                default:
+                    break;
+
+            }
+
+        }
+
+        public void satisfyRequirement(Dpid dpid) {
+            log.warn("Satisfaction from switch {}", dpid);
+            sws.remove(dpid);
+            countDownLatch.countDown();
+        }
 
+        public void verify(Integer id) {
+            for (Dpid dpid : sws) {
+                OpenFlowSwitch sw = controller.getSwitch(dpid);
+                OFBarrierRequest.Builder builder = sw.factory()
+                        .buildBarrierRequest()
+                        .setXid(id);
+                sw.sendMsg(builder.build());
+            }
+
+
+        }
+
+        @Override
+        public boolean cancel(boolean mayInterruptIfRunning) {
+                // TODO Auto-generated method stub
+                return false;
+        }
+
+        @Override
+        public boolean isCancelled() {
+            // TODO Auto-generated method stub
+            return false;
+        }
+
+        @Override
+        public boolean isDone() {
+            return sws.isEmpty();
+        }
+
+        @Override
+        public Void get() throws InterruptedException, ExecutionException {
+            countDownLatch.await();
+            //return offendingFlowMods;
+            return null;
+        }
+
+        @Override
+        public Void get(long timeout, TimeUnit unit)
+                throws InterruptedException, ExecutionException,
+                TimeoutException {
+            countDownLatch.await(timeout, unit);
+            //return offendingFlowMods;
+            return null;
+        }
+
+    }
 
 }

@@ -181,7 +181,7 @@ public class OpenFlowPacketProviderTest {
     }
 
     private static TrafficTreatment treatment(Instruction ... insts) {
-        TrafficTreatment.Builder builder = new DefaultTrafficTreatment.Builder();
+        TrafficTreatment.Builder builder = DefaultTrafficTreatment.builder();
         for (Instruction i : insts) {
             builder.add(i);
         }

@@ -9,10 +9,14 @@ export KARAF_ZIP=${KARAF_ZIP:-~/Downloads/apache-karaf-3.0.1.zip}
 export KARAF_TAR=${KARAF_TAR:-~/Downloads/apache-karaf-3.0.1.tar.gz}
 export KARAF_DIST=$(basename $KARAF_ZIP .zip)
 
+# Fallback build number us derived from from the user name & time
+export BUILD_NUMBER=${BUILD_NUMBER:-$(id -un)~$(date +'%Y/%m/%d@%H:%M')}
+
 # ONOS Version and onos.tar.gz staging environment
-export ONOS_VERSION=${ONOS_VERSION:-1.0.0-SNAPSHOT}
+export ONOS_POM_VERSION="1.0.0-SNAPSHOT"
+export ONOS_VERSION=${ONOS_VERSION:-1.0.0.$BUILD_NUMBER}
+export ONOS_BITS=onos-${ONOS_VERSION%~*}
 export ONOS_STAGE_ROOT=${ONOS_STAGE_ROOT:-/tmp}
-export ONOS_BITS=onos-$ONOS_VERSION
 export ONOS_STAGE=$ONOS_STAGE_ROOT/$ONOS_BITS
 export ONOS_TAR=$ONOS_STAGE.tar.gz
 

@@ -49,7 +49,7 @@ export ONOS_FEATURES="${ONOS_FEATURES:-webconsole,onos-api,onos-core,onos-cli,on
 # ONOS Patching ----------------------------------------------------------------
 
 # Patch the Apache Karaf distribution file to add ONOS features repository
-perl -pi.old -e "s|^(featuresRepositories=.*)|\1,mvn:org.onlab.onos/onos-features/$ONOS_VERSION/xml/features|" \
+perl -pi.old -e "s|^(featuresRepositories=.*)|\1,mvn:org.onlab.onos/onos-features/$ONOS_POM_VERSION/xml/features|" \
     $ONOS_STAGE/$KARAF_DIST/etc/org.apache.karaf.features.cfg 
 
 # Patch the Apache Karaf distribution file to load ONOS features
@@ -57,10 +57,14 @@ perl -pi.old -e "s|^(featuresBoot=.*)|\1,$ONOS_FEATURES|" \
     $ONOS_STAGE/$KARAF_DIST/etc/org.apache.karaf.features.cfg
 
 # Patch the Apache Karaf distribution with ONOS branding bundle
-cp $M2_REPO/org/onlab/onos/onos-branding/$ONOS_VERSION/onos-branding-*.jar \
+cp $M2_REPO/org/onlab/onos/onos-branding/$ONOS_POM_VERSION/onos-branding-*.jar \
     $ONOS_STAGE/$KARAF_DIST/lib
 
+# Patch in the ONOS version file
+echo $ONOS_VERSION > $ONOS_STAGE/VERSION
+
 # Now package up the ONOS tar file
 cd $ONOS_STAGE_ROOT
 COPYFILE_DISABLE=1 tar zcf $ONOS_TAR $ONOS_BITS
 ls -l $ONOS_TAR >&2
+rm -r $ONOS_STAGE

@@ -21,7 +21,7 @@ export PATH="$PATH:."
 # e.g. 'o api', 'o dev', 'o'
 function o {
     cd $(find $ONOS_ROOT/ -type d | egrep -v '\.git|target' | \
-        egrep "${1:-$ONOS_ROOT}" | head -n 1)
+        egrep "${1:-$ONOS_ROOT}" | egrep -v "$ONOS_ROOT/.+/src/" | head -n 1)
 }
 
 # Short-hand for 'mvn clean install' for us lazy folk
@@ -32,6 +32,9 @@ alias ob='onos-build'
 alias obs='onos-build-selective'
 alias op='onos-package'
 alias ot='onos-test'
+alias ol='onos-log'
+alias go='ob && ot && onos -w'
+alias pub='onos-push-update-bundle'
 
 # Short-hand for tailing the ONOS (karaf) log 
 alias tl='$ONOS_ROOT/tools/dev/bin/onos-local-log'
@@ -88,5 +91,5 @@ function spy {
 }
 
 function nuke {
-    spy | cut -c7-11 | xargs kill
+    spy "$@" | cut -c7-11 | xargs kill
 }

@@ -4,6 +4,7 @@
 #-------------------------------------------------------------------------------
 
 export JAVA_HOME=${JAVA_HOME:-/usr/lib/jvm/java-7-openjdk-amd64/}
+export JAVA_OPTS="-Xms256M -Xmx2048M"
 
 cd /opt/onos
 /opt/onos/apache-karaf-3.0.1/bin/karaf "$@"

@@ -32,6 +32,10 @@ ssh $remote "
 
     # Remove any previous ON.Lab bits from ~/.m2 repo
     rm -fr ~/.m2/repository/org/onlab
+
+    # Drop log level for the console
+    echo "log4j.logger.org.apache.sshd = WARN" >> $ONOS_INSTALL_DIR/$KARAF_DIST/etc/org.ops4j.pax.logging.cfg
+
 "
 
 # Configure the ONOS installation

@@ -15,7 +15,7 @@ name=${2:-onos-1}
 
 ssh $remote "
     sudo perl -pi.bak -e \"s/127.0.1.1.*/127.0.1.1       $name/g\" /etc/hosts
-    sudo perl -pi.bak -e \"s/.*/$name/g\" /etc/hostname
+    sudo bash -c \"echo $name >/etc/hostname\"
     sudo hostname $name
 " 2>/dev/null
 

@@ -9,5 +9,9 @@
 remote=$ONOS_USER@${1:-$OCI}
 
 scp -q ~/.ssh/id_rsa.pub $remote:/tmp
-ssh $remote "cat /tmp/id_rsa.pub >> ~/.ssh/authorized_keys"
+ssh $remote "
+    cat /tmp/id_rsa.pub >> ~/.ssh/authorized_keys
+    sort -u ~/.ssh/authorized_keys > ~/.ssh/authorized_keys.bak
+    mv ~/.ssh/authorized_keys.bak ~/.ssh/authorized_keys
+"
 ssh -n -o PasswordAuthentication=no $remote true

+#!/bin/bash
+#-------------------------------------------------------------------------------
+# Pushes the specified bundle to the remote ONOS cell machines and updates it.
+#-------------------------------------------------------------------------------
+
+[ ! -d "$ONOS_ROOT" ] && echo "ONOS_ROOT is not defined" >&2 && exit 1
+. $ONOS_ROOT/tools/build/envDefaults
+
+cd ~/.m2/repository
+jar=$(find org/onlab -type f -name '*.jar' | grep $1 | grep -v -e -tests | head -n 1)
+
+[ -z "$jar" ] && echo "No bundle $1 found for" && exit 1
+
+bundle=$(echo $(basename $jar .jar) | sed 's/-[0-9].*//g')
+
+nodes=$(env | sort | egrep "OC[0-9]+" | cut -d= -f2)
+for node in $nodes; do
+    scp -q $jar $ONOS_USER@$node:.m2/repository/$jar
+    scp -q $jar $ONOS_USER@$node:$ONOS_INSTALL_DIR/$KARAF_DIST/system/$jar
+    ssh $ONOS_USER@$node "$ONOS_INSTALL_DIR/bin/onos \"bundle:update -f $bundle\"" 2>/dev/null
+done

+#!/usr/bin/env python
+from mininet.cli import CLI
+from mininet.net import Mininet
+from mininet.node import RemoteController, OVSKernelSwitch
+
+MAC = 12
+DPID = 16
+
+def string_to_hex(s, length):
+    """ Convert a string like 00:00 in to hex 0x0000 format"""
+    tmp = '{0:#x}'.format(int(s.replace(':', '').lstrip('0'),length))
+    return tmp
+
+def hex_to_string(h, length):
+    """Convert a hex number from 0x0000 to 00:00 format"""
+    tmp = h.lstrip('0x').zfill(length)
+    tmp = ':'.join(a+b for a,b in zip(tmp[::2], tmp[1::2]))
+    return tmp
+
+class Tower(object):
+    """ Create a tower topology from semi-scratch in Mininet """
+
+    def __init__(self, cname='flare', cip='15.255.126.183', k=4, h=6, 
+                 proto=None):
+        """Create tower topology for mininet
+            cname: controller name
+            cip: controller ip
+            k: number of leaf switches
+            h: number of hosts perl leaf switch
+        """
+
+        # We are creating the controller with local-loopback on purpose to avoid
+        # having the switches connect immediately. Instead, we'll set controller
+        # explicitly for each switch after configuring it as we want.
+        self.flare = RemoteController(cname, '127.0.0.1', 6633)
+        self.net = Mininet(controller=self.flare, switch = OVSKernelSwitch, 
+                           build=False)
+
+        self.cip = cip
+        self.spines = []
+        self.leaves = []
+        self.hosts = []
+        self.proto = proto
+
+        # Create the two spine switches
+        self.spines.append(self.net.addSwitch('s1'))
+        self.spines.append(self.net.addSwitch('s2'))
+
+        # Create two links between the spine switches
+        self.net.addLink(self.spines[0], self.spines[1])
+        self.net.addLink(self.spines[1], self.spines[0])
+        
+        # Now create the leaf switches, their hosts and connect them together
+        i = 1
+        c = 0
+        while i <= k:
+            self.leaves.append(self.net.addSwitch('s1%d' % i))
+            for spine in self.spines:
+                self.net.addLink(self.leaves[i-1], spine)
+
+            j = 1
+            while j <= h:
+                self.hosts.append(self.net.addHost('h%d%d' % (i, j)))
+                self.net.addLink(self.hosts[c], self.leaves[i-1])
+                j+=1
+                c+=1
+
+            i+=1
+
+    def run(self):
+        """ Runs the created network topology and launches mininet cli"""
+        self.run_silent()
+        CLI(self.net)
+        self.net.stop()
+
+    def run_silent(self):
+        """ Runs silently - for unit testing """
+        self.net.build()
+
+        # Start the switches, configure them with desired protocols and only 
+        # then set the controller
+        for sw in self.spines:
+            sw.start([self.flare])
+            if self.proto:
+                sw.cmd('ovs-vsctl set bridge %(sw)s protocols=%(proto)s' % \
+                           { 'sw': sw.name, 'proto': self.proto})
+            sw.cmdPrint('ovs-vsctl set-controller %(sw)s tcp:%(ctl)s:6633' % \
+                            {'sw': sw.name, 'ctl': self.cip})
+
+        for sw in self.leaves:
+            sw.start([self.flare])
+            sw.cmdPrint('ovs-vsctl set-controller %(sw)s tcp:%(ctl)s:6633' % \
+                            {'sw': sw.name, 'ctl': self.cip})
+
+    def pingAll(self):
+        """ PingAll to create flows - for unit testing """
+        self.net.pingAll()
+
+    def stop(self):
+        "Stops the topology. You should call this after run_silent"
+        self.net.stop()

+#!/usr/bin/python
+# Launches mininet with Tower topology configuration.
+import sys, tower
+net = tower.Tower(cip=sys.argv[1])
+net.run()

@@ -11,7 +11,7 @@ public class MetricsFeature {
      *
      * @param newName name of the Feature
      */
-    MetricsFeature(final String newName) {
+    public MetricsFeature(final String newName) {
         name = newName;
     }
 

 package org.onlab.metrics;
 
-import java.io.File;
-import java.util.Locale;
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -10,9 +8,11 @@ import java.util.concurrent.TimeUnit;
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
+import com.codahale.metrics.ConsoleReporter;
 import com.codahale.metrics.Counter;
-import com.codahale.metrics.CsvReporter;
 import com.codahale.metrics.Gauge;
 import com.codahale.metrics.Histogram;
 import com.codahale.metrics.Meter;
@@ -56,6 +56,7 @@ import com.codahale.metrics.Timer;
 @Component(immediate = true)
 public final class MetricsManager implements MetricsService {
 
+    private final Logger log = LoggerFactory.getLogger(getClass());
     /**
      * Registry to hold the Components defined in the system.
      */
@@ -69,27 +70,31 @@ public final class MetricsManager implements MetricsService {
     /**
      * Default Reporter for this metrics manager.
      */
-    private final CsvReporter reporter;
+    //private final Slf4jReporter reporter;
+    private final ConsoleReporter reporter;
 
     public MetricsManager() {
-        this.componentsRegistry = new ConcurrentHashMap<>();
         this.metricsRegistry = new MetricRegistry();
-
-        this.reporter = CsvReporter.forRegistry(metricsRegistry)
-                .formatFor(Locale.US)
+//        this.reporter = Slf4jReporter.forRegistry(this.metricsRegistry)
+//                .outputTo(log)
+//                .convertRatesTo(TimeUnit.SECONDS)
+//                .convertDurationsTo(TimeUnit.NANOSECONDS)
+//                .build();
+        this.reporter = ConsoleReporter.forRegistry(this.metricsRegistry)
                 .convertRatesTo(TimeUnit.SECONDS)
-                .convertDurationsTo(TimeUnit.MICROSECONDS)
-                .build(new File("/tmp/"));
-
-        reporter.start(10, TimeUnit.SECONDS);
+                .convertDurationsTo(TimeUnit.NANOSECONDS)
+                .build();
     }
 
     @Activate
     public void activate() {
+        this.componentsRegistry = new ConcurrentHashMap<>();
+        reporter.start(10, TimeUnit.SECONDS);
     }
 
     @Deactivate
     public void deactivate() {
+        reporter.stop();
     }
 
     /**

@@ -22,6 +22,12 @@ import java.util.Arrays;
  *
  */
 public class MacAddress {
+    public static final byte[] ZERO_MAC_ADDRESS =
+            MacAddress.valueOf("00:00:00:00:00:00").getAddress();
+
+    public static final byte[] BROADCAST_MAC =
+            MacAddress.valueOf("ff:ff:ff:ff:ff:ff").getAddress();
+
     public static final int MAC_ADDRESS_LENGTH = 6;
     private byte[] address = new byte[MacAddress.MAC_ADDRESS_LENGTH];
 

+package org.onlab.util;
+
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.commons.lang3.concurrent.ConcurrentException;
+import org.apache.commons.lang3.concurrent.ConcurrentInitializer;
+
+/**
+ * Creates an instance of new ConcurrentHashMap on each {@link #get()} call.
+ * <p>
+ * To be used with
+ * {@link org.apache.commons.lang3.concurrent.ConcurrentUtils#createIfAbsent()
+ *  ConcurrentUtils#createIfAbsent}
+ *
+ * @param <K> ConcurrentHashMap key type
+ * @param <V> ConcurrentHashMap value type
+ */
+public final class NewConcurrentHashMap<K, V>
+    implements  ConcurrentInitializer<ConcurrentMap<K, V>> {
+
+    public static final NewConcurrentHashMap<?, ?> INSTANCE = new NewConcurrentHashMap<>();
+
+    @SuppressWarnings("unchecked")
+    public static <K, V> NewConcurrentHashMap<K, V> ifNeeded() {
+        return (NewConcurrentHashMap<K, V>) INSTANCE;
+    }
+
+    @Override
+    public ConcurrentMap<K, V> get() throws ConcurrentException {
+        return new ConcurrentHashMap<>();
+    }
+}

@@ -8,7 +8,7 @@ import org.jboss.netty.util.HashedWheelTimer;
  */
 public final class Timer {
 
-    private static HashedWheelTimer timer;
+    private static volatile HashedWheelTimer timer;
 
     // Ban public construction
     private Timer() {
@@ -21,10 +21,17 @@ public final class Timer {
      */
     public static HashedWheelTimer getTimer() {
         if (Timer.timer == null) {
-            Timer.timer = new HashedWheelTimer();
-            Timer.timer.start();
+            initTimer();
         }
         return Timer.timer;
     }
 
+    private static synchronized  void initTimer() {
+        if (Timer.timer == null) {
+            HashedWheelTimer hwTimer = new HashedWheelTimer();
+            hwTimer.start();
+            Timer.timer = hwTimer;
+        }
+    }
+
 }

@@ -4,6 +4,12 @@ import com.google.common.base.Strings;
 import com.google.common.primitives.UnsignedLongs;
 import com.google.common.util.concurrent.ThreadFactoryBuilder;
 
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.concurrent.ThreadFactory;
 
 public abstract class Tools {
@@ -66,4 +72,24 @@ public abstract class Tools {
         }
     }
 
+    /**
+     * Slurps the contents of a file into a list of strings, one per line.
+     *
+     * @param path file path
+     * @return file contents
+     */
+    public static List<String> slurp(File path) {
+        try (BufferedReader br = new BufferedReader(new FileReader(path))) {
+            List<String> lines = new ArrayList<>();
+            String line;
+            while ((line = br.readLine()) != null) {
+                lines.add(line);
+            }
+            return lines;
+
+        } catch (IOException e) {
+            return null;
+        }
+    }
+
 }

@@ -8,17 +8,16 @@ import java.util.concurrent.TimeoutException;
  * This class provides a base implementation of Response, with methods to retrieve the
  * result and query to see if the result is ready. The result can only be retrieved when
  * it is ready and the get methods will block if the result is not ready yet.
- * @param <T> type of response.
  */
-public class AsyncResponse<T> implements Response<T> {
+public class AsyncResponse implements Response {
 
-    private T value;
+    private byte[] value;
     private boolean done = false;
     private final long start = System.nanoTime();
 
     @Override
-    public T get(long timeout, TimeUnit tu) throws TimeoutException {
-        timeout = tu.toNanos(timeout);
+    public byte[] get(long timeout, TimeUnit timeUnit) throws TimeoutException {
+        timeout = timeUnit.toNanos(timeout);
         boolean interrupted = false;
         try {
             synchronized (this) {
@@ -43,7 +42,7 @@ public class AsyncResponse<T> implements Response<T> {
     }
 
     @Override
-    public T get() throws InterruptedException {
+    public byte[] get() throws InterruptedException {
         throw new UnsupportedOperationException();
     }
 
@@ -57,11 +56,10 @@ public class AsyncResponse<T> implements Response<T> {
      * available.
      * @param data response data.
      */
-    @SuppressWarnings("unchecked")
-    public synchronized void setResponse(Object data) {
+    public synchronized void setResponse(byte[] data) {
         if (!done) {
             done = true;
-            value = (T) data;
+            value = data;
             this.notifyAll();
         }
     }

+package org.onlab.netty;
+
+/**
+ * State transitions a decoder goes through as it is decoding an incoming message.
+ */
+public enum DecoderState {
+    READ_HEADER_VERSION,
+    READ_PREAMBLE,
+    READ_CONTENT_LENGTH,
+    READ_SERIALIZER_VERSION,
+    READ_CONTENT
+}

@@ -2,14 +2,20 @@ package org.onlab.netty;
 
 import java.io.IOException;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+//FIXME: Should be move out to test or app
 /**
  * Message handler that echos the message back to the sender.
  */
 public class EchoHandler implements MessageHandler {
 
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
     @Override
     public void handle(Message message) throws IOException {
-        System.out.println("Received: " + message.payload() + ". Echoing it back to the sender.");
+        log.info("Received message. Echoing it back to the sender.");
         message.respond(message.payload());
     }
 }

 package org.onlab.netty;
 
+import java.util.Objects;
+
+import com.google.common.base.MoreObjects;
+
 /**
  * Representation of a TCP/UDP communication end point.
  */
@@ -8,6 +12,15 @@ public class Endpoint {
     private final int port;
     private final String host;
 
+    /**
+     * Used for serialization.
+     */
+    @SuppressWarnings("unused")
+    private Endpoint() {
+        port = 0;
+        host = null;
+    }
+
     public Endpoint(String host, int port) {
         this.host = host;
         this.port = port;
@@ -23,16 +36,15 @@ public class Endpoint {
 
     @Override
     public String toString() {
-        return "Endpoint [port=" + port + ", host=" + host + "]";
+        return MoreObjects.toStringHelper(getClass())
+                .add("port", port)
+                .add("host", host)
+                .toString();
     }
 
     @Override
     public int hashCode() {
-        final int prime = 31;
-        int result = 1;
-        result = prime * result + ((host == null) ? 0 : host.hashCode());
-        result = prime * result + port;
-        return result;
+        return Objects.hash(host, port);
     }
 
     @Override
@@ -46,17 +58,8 @@ public class Endpoint {
         if (getClass() != obj.getClass()) {
             return false;
         }
-        Endpoint other = (Endpoint) obj;
-        if (host == null) {
-            if (other.host != null) {
-                return false;
-            }
-        } else if (!host.equals(other.host)) {
-            return false;
-        }
-        if (port != other.port) {
-            return false;
-        }
-        return true;
+        Endpoint that = (Endpoint) obj;
+        return Objects.equals(this.port, that.port) &&
+               Objects.equals(this.host, that.host);
     }
 }

@@ -8,12 +8,13 @@ import java.io.IOException;
  */
 public final class InternalMessage implements Message {
 
+    public static final String REPLY_MESSAGE_TYPE = "NETTY_MESSAGIG_REQUEST_REPLY";
+
     private long id;
     private Endpoint sender;
     private String type;
-    private Object payload;
+    private byte[] payload;
     private transient NettyMessagingService messagingService;
-    public static final String REPLY_MESSAGE_TYPE = "NETTY_MESSAGIG_REQUEST_REPLY";
 
     // Must be created using the Builder.
     private InternalMessage() {}
@@ -31,12 +32,16 @@ public final class InternalMessage implements Message {
     }
 
     @Override
-    public Object payload() {
+    public byte[] payload() {
         return payload;
     }
 
+    protected void setMessagingService(NettyMessagingService messagingService) {
+        this.messagingService = messagingService;
+    }
+
     @Override
-    public void respond(Object data) throws IOException {
+    public void respond(byte[] data) throws IOException {
         Builder builder = new Builder(messagingService);
         InternalMessage message = builder.withId(this.id)
              // FIXME: Sender should be messagingService.localEp.
@@ -51,7 +56,7 @@ public final class InternalMessage implements Message {
     /**
      * Builder for InternalMessages.
      */
-    public static class Builder {
+    public static final class Builder {
         private InternalMessage message;
 
         public Builder(NettyMessagingService messagingService) {
@@ -73,7 +78,7 @@ public final class InternalMessage implements Message {
             message.sender = sender;
             return this;
         }
-        public Builder withPayload(Object payload) {
+        public Builder withPayload(byte[] payload) {
             message.payload = payload;
             return this;
         }

 package org.onlab.netty;
 
 import org.onlab.util.KryoPool;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
+import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.HashMap;
 
+//FIXME: Should be move out to test or app
 /**
  * Kryo Serializer.
  */
-public class KryoSerializer implements Serializer {
-
-    private final Logger log = LoggerFactory.getLogger(getClass());
+public class KryoSerializer {
 
     private KryoPool serializerPool;
 
@@ -28,20 +26,29 @@ public class KryoSerializer implements Serializer {
         serializerPool = KryoPool.newBuilder()
                 .register(ArrayList.class,
                           HashMap.class,
-                          ArrayList.class
+                          ArrayList.class,
+                          InternalMessage.class,
+                          Endpoint.class,
+                          byte[].class
                 )
                 .build()
                 .populate(1);
     }
 
 
-    @Override
-    public Object decode(byte[] data) {
+    public <T> T decode(byte[] data) {
         return serializerPool.deserialize(data);
     }
 
-    @Override
     public byte[] encode(Object payload) {
         return serializerPool.serialize(payload);
     }
+
+    public <T> T decode(ByteBuffer buffer) {
+        return serializerPool.deserialize(buffer);
+    }
+
+    public void encode(Object obj, ByteBuffer buffer) {
+        serializerPool.serialize(obj, buffer);
+    }
 }

 package org.onlab.netty;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 /**
  * A MessageHandler that simply logs the information.
  */
 public class LoggingHandler implements MessageHandler {
 
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
     @Override
     public void handle(Message message) {
-        System.out.println("Received: " + message.payload());
+        log.info("Received message. Payload has {} bytes", message.payload().length);
     }
 }

@@ -12,12 +12,12 @@ public interface Message {
      * Returns the payload of this message.
      * @return message payload.
      */
-    public Object payload();
+    public byte[] payload();
 
     /**
-     * Sends a reply back to the sender of this messge.
+     * Sends a reply back to the sender of this message.
      * @param data payload of the response.
      * @throws IOException if there is a communication error.
      */
-    public void respond(Object data) throws IOException;
+    public void respond(byte[] data) throws IOException;
 }

 package org.onlab.netty;
 
-import java.util.Arrays;
-import java.util.List;
-
 import static com.google.common.base.Preconditions.checkState;
-
 import io.netty.buffer.ByteBuf;
 import io.netty.channel.ChannelHandlerContext;
-import io.netty.handler.codec.ByteToMessageDecoder;
+import io.netty.handler.codec.ReplayingDecoder;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 /**
- * Decode bytes into a InternalMessage.
+ * Decoder for inbound messages.
  */
-public class MessageDecoder extends ByteToMessageDecoder {
+public class MessageDecoder extends ReplayingDecoder<DecoderState> {
+
+    private final Logger log = LoggerFactory.getLogger(getClass());
 
     private final NettyMessagingService messagingService;
-    private final Serializer serializer;
 
-    public MessageDecoder(NettyMessagingService messagingService, Serializer serializer) {
+    private static final KryoSerializer SERIALIZER = new KryoSerializer();
+
+    private int contentLength;
+
+    public MessageDecoder(NettyMessagingService messagingService) {
+        super(DecoderState.READ_HEADER_VERSION);
         this.messagingService = messagingService;
-        this.serializer = serializer;
     }
 
     @Override
-    protected void decode(ChannelHandlerContext context, ByteBuf in,
-            List<Object> messages) throws Exception {
-
-        byte[] preamble = in.readBytes(MessageEncoder.PREAMBLE.length).array();
-        checkState(Arrays.equals(MessageEncoder.PREAMBLE, preamble), "Message has wrong preamble");
-
-        // read message Id.
-        long id = in.readLong();
-
-        // read message type; first read size and then bytes.
-        String type = new String(in.readBytes(in.readInt()).array());
-
-        // read sender host name; first read size and then bytes.
-        String host = new String(in.readBytes(in.readInt()).array());
-
-        // read sender port.
-        int port = in.readInt();
-
-        Endpoint sender = new Endpoint(host, port);
-
-        // read message payload; first read size and then bytes.
-        Object payload = serializer.decode(in.readBytes(in.readInt()).array());
-
-        InternalMessage message = new InternalMessage.Builder(messagingService)
-                .withId(id)
-                .withSender(sender)
-                .withType(type)
-                .withPayload(payload)
-                .build();
+    protected void decode(
+            ChannelHandlerContext context,
+            ByteBuf buffer,
+            List<Object> out) throws Exception {
+
+        switch(state()) {
+        case READ_HEADER_VERSION:
+            int headerVersion = buffer.readInt();
+            checkState(headerVersion == MessageEncoder.HEADER_VERSION, "Unexpected header version");
+            checkpoint(DecoderState.READ_PREAMBLE);
+        case READ_PREAMBLE:
+            byte[] preamble = new byte[MessageEncoder.PREAMBLE.length];
+            buffer.readBytes(preamble);
+            checkState(Arrays.equals(MessageEncoder.PREAMBLE, preamble), "Message has wrong preamble");
+            checkpoint(DecoderState.READ_CONTENT_LENGTH);
+        case READ_CONTENT_LENGTH:
+            contentLength = buffer.readInt();
+            checkpoint(DecoderState.READ_SERIALIZER_VERSION);
+        case READ_SERIALIZER_VERSION:
+            int serializerVersion = buffer.readInt();
+            checkState(serializerVersion == MessageEncoder.SERIALIZER_VERSION, "Unexpected serializer version");
+            checkpoint(DecoderState.READ_CONTENT);
+        case READ_CONTENT:
+            InternalMessage message = SERIALIZER.decode(buffer.readBytes(contentLength).nioBuffer());
+            message.setMessagingService(messagingService);
+            out.add(message);
+            checkpoint(DecoderState.READ_HEADER_VERSION);
+            break;
+         default:
+            checkState(false, "Must not be here");
+        }
+    }
 
-        messages.add(message);
+    @Override
+    public void exceptionCaught(ChannelHandlerContext context, Throwable cause) {
+        log.error("Exception inside channel handling pipeline.", cause);
+        context.close();
     }
 }

 package org.onlab.netty;
 
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
 import io.netty.buffer.ByteBuf;
+import io.netty.channel.ChannelHandler.Sharable;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.handler.codec.MessageToByteEncoder;
 
 /**
  * Encode InternalMessage out into a byte buffer.
  */
+@Sharable
 public class MessageEncoder extends MessageToByteEncoder<InternalMessage> {
 
+    private final Logger log = LoggerFactory.getLogger(getClass());
+
     // onosiscool in ascii
     public static final byte[] PREAMBLE = "onosiscool".getBytes();
+    public static final int HEADER_VERSION = 1;
+    public static final int SERIALIZER_VERSION = 1;
 
-    private final Serializer serializer;
 
-    public MessageEncoder(Serializer serializer) {
-        this.serializer = serializer;
-    }
+    private static final KryoSerializer SERIALIZER = new KryoSerializer();
 
     @Override
-    protected void encode(ChannelHandlerContext context, InternalMessage message,
+    protected void encode(
+            ChannelHandlerContext context,
+            InternalMessage message,
             ByteBuf out) throws Exception {
 
+        // write version
+        out.writeInt(HEADER_VERSION);
+
         // write preamble
         out.writeBytes(PREAMBLE);
 
-        // write id
-        out.writeLong(message.id());
-
-        // write type length
-        out.writeInt(message.type().length());
-
-        // write type
-        out.writeBytes(message.type().getBytes());
-
-        // write sender host name size
-        out.writeInt(message.sender().host().length());
-
-        // write sender host name.
-        out.writeBytes(message.sender().host().getBytes());
+        byte[] payload = SERIALIZER.encode(message);
 
-        // write port
-        out.writeInt(message.sender().port());
-
-        try {
-            serializer.encode(message.payload());
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-
-        byte[] payload = serializer.encode(message.payload());
-
-        // write payload length.
+        // write payload length
         out.writeInt(payload.length);
 
-        // write payload bytes
+        // write payloadSerializer version
+        out.writeInt(SERIALIZER_VERSION);
+
+        // write payload.
         out.writeBytes(payload);
     }
+
+    @Override
+    public void exceptionCaught(ChannelHandlerContext context, Throwable cause) {
+        log.error("Exception inside channel handling pipeline.", cause);
+        context.close();
+    }
 }

@@ -11,10 +11,10 @@ public interface MessagingService {
      * The message is specified using the type and payload.
      * @param ep end point to send the message to.
      * @param type type of message.
-     * @param payload message payload.
+     * @param payload message payload bytes.
      * @throws IOException
      */
-    public void sendAsync(Endpoint ep, String type, Object payload) throws IOException;
+    public void sendAsync(Endpoint ep, String type, byte[] payload) throws IOException;
 
     /**
      * Sends a message synchronously and waits for a response.
@@ -24,7 +24,7 @@ public interface MessagingService {
      * @return a response future
      * @throws IOException
      */
-    public <T> Response<T> sendAndReceive(Endpoint ep, String type, Object payload) throws IOException;
+    public Response sendAndReceive(Endpoint ep, String type, byte[] payload) throws IOException;
 
     /**
      * Registers a new message handler for message type.
@@ -38,4 +38,4 @@ public interface MessagingService {
      * @param type message type
      */
     public void unregisterHandler(String type);
-}
+}
\ No newline at end of file

@@ -11,6 +11,7 @@ import io.netty.bootstrap.ServerBootstrap;
 import io.netty.buffer.PooledByteBufAllocator;
 import io.netty.channel.Channel;
 import io.netty.channel.ChannelFuture;
+import io.netty.channel.ChannelHandler;
 import io.netty.channel.ChannelHandlerContext;
 import io.netty.channel.ChannelInitializer;
 import io.netty.channel.ChannelOption;
@@ -22,7 +23,6 @@ import io.netty.channel.socket.nio.NioServerSocketChannel;
 import io.netty.channel.socket.nio.NioSocketChannel;
 
 import org.apache.commons.lang.math.RandomUtils;
-import org.apache.commons.pool.KeyedObjectPool;
 import org.apache.commons.pool.KeyedPoolableObjectFactory;
 import org.apache.commons.pool.impl.GenericKeyedObjectPool;
 import org.slf4j.Logger;
@@ -38,16 +38,19 @@ public class NettyMessagingService implements MessagingService {
 
     private final Logger log = LoggerFactory.getLogger(getClass());
 
-    private KeyedObjectPool<Endpoint, Channel> channels =
-            new GenericKeyedObjectPool<Endpoint, Channel>(new OnosCommunicationChannelFactory());
     private final int port;
+    private final Endpoint localEp;
     private final EventLoopGroup bossGroup = new NioEventLoopGroup();
     private final EventLoopGroup workerGroup = new NioEventLoopGroup();
     private final ConcurrentMap<String, MessageHandler> handlers = new ConcurrentHashMap<>();
-    private Cache<Long, AsyncResponse<?>> responseFutures;
-    private final Endpoint localEp;
-
-    protected Serializer serializer;
+    private final Cache<Long, AsyncResponse> responseFutures = CacheBuilder.newBuilder()
+            .maximumSize(100000)
+            .weakValues()
+            // TODO: Once the entry expires, notify blocking threads (if any).
+            .expireAfterWrite(10, TimeUnit.MINUTES)
+            .build();
+    private final GenericKeyedObjectPool<Endpoint, Channel> channels
+            = new GenericKeyedObjectPool<Endpoint, Channel>(new OnosCommunicationChannelFactory());
 
     public NettyMessagingService() {
         // TODO: Default port should be configurable.
@@ -66,12 +69,8 @@ public class NettyMessagingService implements MessagingService {
     }
 
     public void activate() throws Exception {
-        responseFutures = CacheBuilder.newBuilder()
-                .maximumSize(100000)
-                .weakValues()
-                // TODO: Once the entry expires, notify blocking threads (if any).
-                .expireAfterWrite(10, TimeUnit.MINUTES)
-                .build();
+        channels.setTestOnBorrow(true);
+        channels.setTestOnReturn(true);
         startAcceptingConnections();
     }
 
@@ -82,7 +81,7 @@ public class NettyMessagingService implements MessagingService {
     }
 
     @Override
-    public void sendAsync(Endpoint ep, String type, Object payload) throws IOException {
+    public void sendAsync(Endpoint ep, String type, byte[] payload) throws IOException {
         InternalMessage message = new InternalMessage.Builder(this)
             .withId(RandomUtils.nextLong())
             .withSender(localEp)
@@ -95,24 +94,21 @@ public class NettyMessagingService implements MessagingService {
     protected void sendAsync(Endpoint ep, InternalMessage message) throws IOException {
         Channel channel = null;
         try {
-            channel = channels.borrowObject(ep);
-            channel.eventLoop().execute(new WriteTask(channel, message));
-        } catch (Exception e) {
-            throw new IOException(e);
-        } finally {
             try {
+                channel = channels.borrowObject(ep);
+                channel.eventLoop().execute(new WriteTask(channel, message));
+            } finally {
                 channels.returnObject(ep, channel);
-            } catch (Exception e) {
-                log.warn("Error returning object back to the pool", e);
-                // ignored.
             }
+        } catch (Exception e) {
+            throw new IOException(e);
         }
     }
 
     @Override
-    public <T> Response<T> sendAndReceive(Endpoint ep, String type, Object payload)
+    public Response sendAndReceive(Endpoint ep, String type, byte[] payload)
             throws IOException {
-        AsyncResponse<T> futureResponse = new AsyncResponse<T>();
+        AsyncResponse futureResponse = new AsyncResponse();
         Long messageId = RandomUtils.nextLong();
         responseFutures.put(messageId, futureResponse);
         InternalMessage message = new InternalMessage.Builder(this)
@@ -141,6 +137,9 @@ public class NettyMessagingService implements MessagingService {
 
     private void startAcceptingConnections() throws InterruptedException {
         ServerBootstrap b = new ServerBootstrap();
+        b.option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, 32 * 1024);
+        b.option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, 8 * 1024);
+        // TODO: Need JVM options to configure PooledByteBufAllocator.
         b.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
         b.group(bossGroup, workerGroup)
             .channel(NioServerSocketChannel.class)
@@ -167,17 +166,18 @@ public class NettyMessagingService implements MessagingService {
 
         @Override
         public Channel makeObject(Endpoint ep) throws Exception {
-            Bootstrap b = new Bootstrap();
-            b.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
-            b.group(workerGroup);
+            Bootstrap bootstrap = new Bootstrap();
+            bootstrap.option(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT);
+            bootstrap.option(ChannelOption.WRITE_BUFFER_HIGH_WATER_MARK, 32 * 1024);
+            bootstrap.option(ChannelOption.WRITE_BUFFER_LOW_WATER_MARK, 8 * 1024);
+            bootstrap.group(workerGroup);
             // TODO: Make this faster:
             // http://normanmaurer.me/presentations/2014-facebook-eng-netty/slides.html#37.0
-            b.channel(NioSocketChannel.class);
-            b.option(ChannelOption.SO_KEEPALIVE, true);
-            b.handler(new OnosCommunicationChannelInitializer());
-
+            bootstrap.channel(NioSocketChannel.class);
+            bootstrap.option(ChannelOption.SO_KEEPALIVE, true);
+            bootstrap.handler(new OnosCommunicationChannelInitializer());
             // Start the client.
-            ChannelFuture f = b.connect(ep.host(), ep.port()).sync();
+            ChannelFuture f = bootstrap.connect(ep.host(), ep.port()).sync();
             return f.channel();
         }
 
@@ -194,31 +194,35 @@ public class NettyMessagingService implements MessagingService {
 
     private class OnosCommunicationChannelInitializer extends ChannelInitializer<SocketChannel> {
 
+        private final ChannelHandler dispatcher = new InboundMessageDispatcher();
+        private final ChannelHandler encoder = new MessageEncoder();
+
         @Override
         protected void initChannel(SocketChannel channel) throws Exception {
             channel.pipeline()
-                .addLast(new MessageEncoder(serializer))
-                .addLast(new MessageDecoder(NettyMessagingService.this, serializer))
-                .addLast(new NettyMessagingService.InboundMessageDispatcher());
+                .addLast("encoder", encoder)
+                .addLast("decoder", new MessageDecoder(NettyMessagingService.this))
+                .addLast("handler", dispatcher);
         }
     }
 
     private class WriteTask implements Runnable {
 
-        private final Object message;
+        private final InternalMessage message;
         private final Channel channel;
 
-        public WriteTask(Channel channel, Object message) {
-            this.message = message;
+        public WriteTask(Channel channel, InternalMessage message) {
             this.channel = channel;
+            this.message = message;
         }
 
         @Override
         public void run() {
-            channel.writeAndFlush(message);
+            channel.writeAndFlush(message, channel.voidPromise());
         }
     }
 
+    @ChannelHandler.Sharable
     private class InboundMessageDispatcher extends SimpleChannelInboundHandler<InternalMessage> {
 
         @Override
@@ -226,12 +230,13 @@ public class NettyMessagingService implements MessagingService {
             String type = message.type();
             if (type.equals(InternalMessage.REPLY_MESSAGE_TYPE)) {
                 try {
-                    AsyncResponse<?> futureResponse =
+                    AsyncResponse futureResponse =
                         NettyMessagingService.this.responseFutures.getIfPresent(message.id());
                     if (futureResponse != null) {
                         futureResponse.setResponse(message.payload());
+                    } else {
+                        log.warn("Received a reply. But was unable to locate the request handle");
                     }
-                    log.warn("Received a reply. But was unable to locate the request handle");
                 } finally {
                     NettyMessagingService.this.responseFutures.invalidate(message.id());
                 }
@@ -240,5 +245,11 @@ public class NettyMessagingService implements MessagingService {
             MessageHandler handler = NettyMessagingService.this.getMessageHandler(type);
             handler.handle(message);
         }
+
+        @Override
+        public void exceptionCaught(ChannelHandlerContext context, Throwable cause) {
+            log.error("Exception inside channel handling pipeline.", cause);
+            context.close();
+        }
     }
 }

@@ -7,26 +7,24 @@ import java.util.concurrent.TimeoutException;
  * Response object returned when making synchronous requests.
  * Can you used to check is a response is ready and/or wait for a response
  * to become available.
- *
- * @param <T> type of response.
  */
-public interface Response<T> {
+public interface Response {
 
     /**
      * Gets the response waiting for a designated timeout period.
      * @param timeout timeout period (since request was sent out)
      * @param tu unit of time.
-     * @return response
+     * @return response payload
      * @throws TimeoutException if the timeout expires before the response arrives.
      */
-    public T get(long timeout, TimeUnit tu) throws TimeoutException;
+    public byte[] get(long timeout, TimeUnit tu) throws TimeoutException;
 
     /**
      * Gets the response waiting for indefinite timeout period.
-     * @return response
+     * @return response payload
      * @throws InterruptedException if the thread is interrupted before the response arrives.
      */
-    public T get() throws InterruptedException;
+    public byte[] get() throws InterruptedException;
 
     /**
      * Checks if the response is ready without blocking.

-package org.onlab.netty;
-
-/**
- * Interface for encoding/decoding message payloads.
- */
-public interface Serializer {
-
-    /**
-     * Decodes the specified byte array to a POJO.
-     *
-     * @param data byte array.
-     * @return POJO
-     */
-    Object decode(byte[] data);
-
-    /**
-     * Encodes the specified POJO into a byte array.
-     *
-     * @param data POJO to be encoded
-     * @return byte array.
-     */
-    byte[] encode(Object message);
-
-}

-package org.onlab.netty;
-
-import java.util.concurrent.TimeUnit;
-
-public final class SimpleClient {
-    private SimpleClient() {}
-
-    public static void main(String... args) throws Exception {
-        NettyMessagingService messaging = new TestNettyMessagingService(9081);
-        messaging.activate();
-
-        messaging.sendAsync(new Endpoint("localhost", 8080), "simple", "Hello World");
-        Response<String> response = messaging.sendAndReceive(new Endpoint("localhost", 8080), "echo", "Hello World");
-        System.out.println("Got back:" + response.get(2, TimeUnit.SECONDS));
-    }
-
-    public static class TestNettyMessagingService extends NettyMessagingService {
-        public TestNettyMessagingService(int port) throws Exception {
-            super(port);
-            Serializer serializer = new KryoSerializer();
-            this.serializer = serializer;
-        }
-    }
-}

-package org.onlab.netty;
-
-public final class SimpleServer {
-    private SimpleServer() {}
-
-    public static void main(String... args) throws Exception {
-        NettyMessagingService server = new TestNettyMessagingService();
-        server.activate();
-        server.registerHandler("simple", new LoggingHandler());
-        server.registerHandler("echo", new EchoHandler());
-    }
-
-    public static class TestNettyMessagingService extends NettyMessagingService {
-        protected TestNettyMessagingService() {
-            Serializer serializer = new KryoSerializer();
-            this.serializer = serializer;
-        }
-    }
-}

 /**
  * Asynchronous messaging APIs implemented using the Netty framework.
  */
-package org.onlab.netty;
\ No newline at end of file
+package org.onlab.netty;

+package org.onlab.netty;
+
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.lang3.RandomUtils;
+import static org.junit.Assert.*;
+import org.junit.Test;
+
+/**
+ * Simple ping-pong test that exercises NettyMessagingService.
+ */
+public class PingPongTest {
+
+    @Test
+    public void testPingPong() throws Exception {
+        NettyMessagingService pinger = new NettyMessagingService(8085);
+        NettyMessagingService ponger = new NettyMessagingService(9086);
+        try {
+            pinger.activate();
+            ponger.activate();
+            ponger.registerHandler("echo", new EchoHandler());
+            byte[] payload = RandomUtils.nextBytes(100);
+            Response response = pinger.sendAndReceive(new Endpoint("localhost", 9086), "echo", payload);
+            assertArrayEquals(payload, response.get(10000, TimeUnit.MILLISECONDS));
+        } finally {
+            pinger.deactivate();
+            ponger.deactivate();
+        }
+    }
+}