[Falcon][ONOS-3537] Implement control message collecting logic w/ unit test

Change-Id: Ic21d476a5ad92d7ef739fa3c13dcc06e5cbf7c56

@@ -157,4 +157,18 @@ public interface OpenFlowSwitch {
      * @return string representation of the connection to the device
      */
     String channelId();
+
+    /**
+     * Registers a listener for OF msg events.
+     *
+     * @param listener the listener to notify
+     */
+    void addEventListener(OpenFlowEventListener listener);
+
+    /**
+     * Unregisters a listener.
+     *
+     * @param listener the listener to unregister
+     */
+    void removeEventListener(OpenFlowEventListener listener);
 }

@@ -16,27 +16,33 @@
 
 package org.onosproject.openflow.controller.driver;
 
+import static org.onlab.util.Tools.groupedThreads;
+
 import com.google.common.collect.Lists;
 import org.jboss.netty.channel.Channel;
 import org.onlab.packet.IpAddress;
 import org.onosproject.net.Device;
 import org.onosproject.net.driver.AbstractHandlerBehaviour;
 import org.onosproject.openflow.controller.Dpid;
+import org.onosproject.openflow.controller.OpenFlowEventListener;
 import org.onosproject.openflow.controller.RoleState;
+
 import org.projectfloodlight.openflow.protocol.OFDescStatsReply;
-import org.projectfloodlight.openflow.protocol.OFErrorMsg;
-import org.projectfloodlight.openflow.protocol.OFExperimenter;
-import org.projectfloodlight.openflow.protocol.OFFactories;
-import org.projectfloodlight.openflow.protocol.OFFactory;
 import org.projectfloodlight.openflow.protocol.OFFeaturesReply;
+import org.projectfloodlight.openflow.protocol.OFPortDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFVersion;
 import org.projectfloodlight.openflow.protocol.OFMessage;
-import org.projectfloodlight.openflow.protocol.OFNiciraControllerRoleRequest;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFFactories;
 import org.projectfloodlight.openflow.protocol.OFPortDesc;
-import org.projectfloodlight.openflow.protocol.OFPortDescStatsReply;
+import org.projectfloodlight.openflow.protocol.OFExperimenter;
+import org.projectfloodlight.openflow.protocol.OFErrorMsg;
+import org.projectfloodlight.openflow.protocol.OFRoleRequest;
+import org.projectfloodlight.openflow.protocol.OFNiciraControllerRoleRequest;
 import org.projectfloodlight.openflow.protocol.OFPortStatus;
+import org.projectfloodlight.openflow.protocol.OFFactory;
 import org.projectfloodlight.openflow.protocol.OFRoleReply;
-import org.projectfloodlight.openflow.protocol.OFRoleRequest;
-import org.projectfloodlight.openflow.protocol.OFVersion;
+
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -46,6 +52,10 @@ import java.net.SocketAddress;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
+import java.util.Set;
+import java.util.concurrent.CopyOnWriteArraySet;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.stream.Collectors;
@@ -82,6 +92,11 @@ public abstract class AbstractOpenFlowSwitch extends AbstractHandlerBehaviour
     protected OFFeaturesReply features;
     protected OFDescStatsReply desc;
 
+    protected Set<OpenFlowEventListener> ofEventListener = new CopyOnWriteArraySet<>();
+
+    protected ExecutorService executorMsgs =
+            Executors.newFixedThreadPool(2, groupedThreads("onos/of", "ctrl-msg-stats-%d"));
+
     private final AtomicReference<List<OFMessage>> messagesPendingMastership
             = new AtomicReference<>();
 
@@ -148,6 +163,15 @@ public abstract class AbstractOpenFlowSwitch extends AbstractHandlerBehaviour
                          dpid, role, channel.isConnected(), msgs);
             }
         }
+
+        // listen to outgoing control messages
+        msgs.forEach(m -> {
+            if (m.getType() == OFType.PACKET_OUT ||
+                m.getType() == OFType.FLOW_MOD ||
+                m.getType() == OFType.STATS_REQUEST) {
+                executorMsgs.submit(new OFMessageHandler(dpid, m));
+            }
+        });
     }
 
     private void sendMsgsOnChannel(List<OFMessage> msgs) {
@@ -301,6 +325,16 @@ public abstract class AbstractOpenFlowSwitch extends AbstractHandlerBehaviour
     }
 
     @Override
+    public void addEventListener(OpenFlowEventListener listener) {
+        ofEventListener.add(listener);
+    }
+
+    @Override
+    public void removeEventListener(OpenFlowEventListener listener) {
+        ofEventListener.remove(listener);
+    }
+
+    @Override
     public OFFactory factory() {
         return OFFactories.getFactory(ofVersion);
     }
@@ -491,4 +525,25 @@ public abstract class AbstractOpenFlowSwitch extends AbstractHandlerBehaviour
                 ? channel.getRemoteAddress() : "?")
                 + " DPID[" + ((getStringId() != null) ? getStringId() : "?") + "]]";
     }
+
+    /**
+     * OpenFlow message handler for outgoing control messages.
+     */
+    protected final class OFMessageHandler implements Runnable {
+
+        protected final OFMessage msg;
+        protected final Dpid dpid;
+
+        public OFMessageHandler(Dpid dpid, OFMessage msg) {
+            this.msg = msg;
+            this.dpid = dpid;
+        }
+
+        @Override
+        public void run() {
+            for (OpenFlowEventListener listener : ofEventListener) {
+                listener.handleMessage(dpid, msg);
+            }
+        }
+    }
 }

@@ -217,5 +217,4 @@ public interface OpenFlowSwitchDriver extends OpenFlowSwitch, HandlerBehaviour {
      * @param message an OpenFlow message
      */
     void sendHandshakeMessage(OFMessage message);
-
 }

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openflow.controller.driver;
+
+import org.junit.Before;
+import org.junit.Test;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.jboss.netty.channel.Channel;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+
+/**
+ * Tests for packet processing in the abstract openflow switch class.
+ */
+public class AbstractOpenFlowSwitchTest {
+
+    OpenFlowSwitchImpl ofSwitch;
+    TestExecutorService executorService;
+
+    /**
+     * Mock executor service that tracks submits.
+     */
+    static class TestExecutorService extends ExecutorServiceAdapter {
+        private List<OFMessage> submittedMessages = new ArrayList<>();
+
+        List<OFMessage> submittedMessages() {
+            return submittedMessages;
+        }
+
+        @Override
+        public Future<?> submit(Runnable task) {
+            AbstractOpenFlowSwitch.OFMessageHandler handler =
+                    (AbstractOpenFlowSwitch.OFMessageHandler) task;
+            submittedMessages.add(handler.msg);
+            return null;
+        }
+    }
+
+    /**
+     * Sets up switches to use as data.
+     */
+    @Before
+    public void setUp() {
+        ofSwitch = new OpenFlowSwitchImpl();
+
+        executorService = new TestExecutorService();
+        ofSwitch.executorMsgs = executorService;
+        Channel channel = new ChannelAdapter();
+        ofSwitch.setChannel(channel);
+    }
+
+    /**
+     * Tests a packet out operation.
+     */
+    @Test
+    public void testPacketOut() {
+        OFMessage ofPacketOut = new MockOfPacketOut();
+        ofSwitch.sendMsg(ofPacketOut);
+        assertThat(executorService.submittedMessages(), hasSize(1));
+        assertThat(executorService.submittedMessages().get(0), is(ofPacketOut));
+    }
+
+    /**
+     * Tests a flow mod operation.
+     */
+    @Test
+    public void testFlowMod() {
+        OFMessage ofFlowMod = new MockOfFlowMod();
+        ofSwitch.sendMsg(ofFlowMod);
+        assertThat(executorService.submittedMessages(), hasSize(1));
+        assertThat(executorService.submittedMessages().get(0), is(ofFlowMod));
+    }
+
+    /**
+     * Tests a stats request operation.
+     */
+    @Test
+    public void testStatsRequest() {
+        OFMessage ofStatsRequest = new MockOfStatsRequest();
+        ofSwitch.sendMsg(ofStatsRequest);
+        assertThat(executorService.submittedMessages(), hasSize(1));
+        assertThat(executorService.submittedMessages().get(0), is(ofStatsRequest));
+    }
+
+    protected class OpenFlowSwitchImpl extends AbstractOpenFlowSwitch {
+
+        @Override
+        public Boolean supportNxRole() {
+            return null;
+        }
+
+        @Override
+        public void startDriverHandshake() {
+        }
+
+        @Override
+        public boolean isDriverHandshakeComplete() {
+            return false;
+        }
+
+        @Override
+        public void processDriverHandshakeMessage(OFMessage m) {
+        }
+    }
+}

+/*
+ * Copyright 2014-2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openflow.controller.driver;
+
+import java.net.SocketAddress;
+
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelConfig;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelPipeline;
+
+/**
+ * Adapter for testing against a netty channel.
+ */
+public class ChannelAdapter implements Channel {
+    @Override
+    public Integer getId() {
+        return null;
+    }
+
+    @Override
+    public ChannelFactory getFactory() {
+        return null;
+    }
+
+    @Override
+    public Channel getParent() {
+        return null;
+    }
+
+    @Override
+    public ChannelConfig getConfig() {
+        return null;
+    }
+
+    @Override
+    public ChannelPipeline getPipeline() {
+        return null;
+    }
+
+    @Override
+    public boolean isOpen() {
+        return false;
+    }
+
+    @Override
+    public boolean isBound() {
+        return false;
+    }
+
+    @Override
+    public boolean isConnected() {
+        return false;
+    }
+
+    @Override
+    public SocketAddress getLocalAddress() {
+        return null;
+    }
+
+    @Override
+    public SocketAddress getRemoteAddress() {
+        return null;
+    }
+
+    @Override
+    public ChannelFuture write(Object o) {
+        return null;
+    }
+
+    @Override
+    public ChannelFuture write(Object o, SocketAddress socketAddress) {
+        return null;
+    }
+
+    @Override
+    public ChannelFuture bind(SocketAddress socketAddress) {
+        return null;
+    }
+
+    @Override
+    public ChannelFuture connect(SocketAddress socketAddress) {
+        return null;
+    }
+
+    @Override
+    public ChannelFuture disconnect() {
+        return null;
+    }
+
+    @Override
+    public ChannelFuture unbind() {
+        return null;
+    }
+
+    @Override
+    public ChannelFuture close() {
+        return null;
+    }
+
+    @Override
+    public ChannelFuture getCloseFuture() {
+        return null;
+    }
+
+    @Override
+    public int getInterestOps() {
+        return 0;
+    }
+
+    @Override
+    public boolean isReadable() {
+        return false;
+    }
+
+    @Override
+    public boolean isWritable() {
+        return false;
+    }
+
+    @Override
+    public ChannelFuture setInterestOps(int i) {
+        return null;
+    }
+
+    @Override
+    public ChannelFuture setReadable(boolean b) {
+        return null;
+    }
+
+    @Override
+    public Object getAttachment() {
+        return null;
+    }
+
+    @Override
+    public void setAttachment(Object o) {
+
+    }
+
+    @Override
+    public int compareTo(Channel o) {
+        return 0;
+    }
+}
\ No newline at end of file

+/*
+ * Copyright 2014-2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openflow.controller.driver;
+
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Test harness adapter for the ExecutorService.
+ */
+public class ExecutorServiceAdapter  implements ExecutorService {
+    @Override
+    public void shutdown() {
+
+    }
+
+    @Override
+    public List<Runnable> shutdownNow() {
+        return null;
+    }
+
+    @Override
+    public boolean isShutdown() {
+        return false;
+    }
+
+    @Override
+    public boolean isTerminated() {
+        return false;
+    }
+
+    @Override
+    public boolean awaitTermination(long timeout, TimeUnit unit)
+            throws InterruptedException {
+        return false;
+    }
+
+    @Override
+    public <T> Future<T> submit(Callable<T> task) {
+        return null;
+    }
+
+    @Override
+    public <T> Future<T> submit(Runnable task, T result) {
+        return null;
+    }
+
+    @Override
+    public Future<?> submit(Runnable task) {
+        return null;
+    }
+
+    @Override
+    public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks)
+            throws InterruptedException {
+        return null;
+    }
+
+    @Override
+    public <T> List<Future<T>> invokeAll(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit)
+            throws InterruptedException {
+        return null;
+    }
+
+    @Override
+    public <T> T invokeAny(Collection<? extends Callable<T>> tasks) throws InterruptedException, ExecutionException {
+        return null;
+    }
+
+    @Override
+    public <T> T invokeAny(Collection<? extends Callable<T>> tasks, long timeout, TimeUnit unit)
+            throws InterruptedException, ExecutionException, TimeoutException {
+        return null;
+    }
+
+    @Override
+    public void execute(Runnable command) {
+
+    }
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openflow.controller.driver;
+
+
+import org.projectfloodlight.openflow.protocol.OFFlowMod;
+import org.projectfloodlight.openflow.protocol.OFFlowModCommand;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFFlowModFlags;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+import org.projectfloodlight.openflow.protocol.instruction.OFInstruction;
+import org.projectfloodlight.openflow.protocol.match.Match;
+import org.projectfloodlight.openflow.types.OFBufferId;
+import org.projectfloodlight.openflow.types.OFPort;
+import org.projectfloodlight.openflow.types.TableId;
+import org.projectfloodlight.openflow.types.U64;
+import org.projectfloodlight.openflow.types.OFGroup;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Mock of the Open Flow flow mod message.
+ */
+public class MockOfFlowMod extends OfMessageAdapter implements OFFlowMod {
+
+    public MockOfFlowMod() {
+        super(OFType.FLOW_MOD);
+    }
+
+    @Override
+    public U64 getCookie() {
+        return null;
+    }
+
+    @Override
+    public U64 getCookieMask() throws UnsupportedOperationException {
+        return null;
+    }
+
+    @Override
+    public TableId getTableId() throws UnsupportedOperationException {
+        return null;
+    }
+
+    @Override
+    public OFFlowModCommand getCommand() {
+        return null;
+    }
+
+    @Override
+    public int getIdleTimeout() {
+        return 0;
+    }
+
+    @Override
+    public int getHardTimeout() {
+        return 0;
+    }
+
+    @Override
+    public int getPriority() {
+        return 0;
+    }
+
+    @Override
+    public OFBufferId getBufferId() {
+        return null;
+    }
+
+    @Override
+    public OFPort getOutPort() {
+        return null;
+    }
+
+    @Override
+    public OFGroup getOutGroup() throws UnsupportedOperationException {
+        return null;
+    }
+
+    @Override
+    public Set<OFFlowModFlags> getFlags() {
+        return null;
+    }
+
+    @Override
+    public Match getMatch() {
+        return null;
+    }
+
+    @Override
+    public List<OFInstruction> getInstructions() throws UnsupportedOperationException {
+        return null;
+    }
+
+    @Override
+    public List<OFAction> getActions() throws UnsupportedOperationException {
+        return null;
+    }
+
+    @Override
+    public OFFlowMod.Builder createBuilder() {
+        return null;
+    }
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openflow.controller.driver;
+
+import org.projectfloodlight.openflow.protocol.OFPacketOut;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.action.OFAction;
+import org.projectfloodlight.openflow.types.OFBufferId;
+import org.projectfloodlight.openflow.types.OFPort;
+
+import java.util.List;
+
+/**
+ * Mock of the Open Flow packet out message.
+ */
+public class MockOfPacketOut extends OfMessageAdapter implements OFPacketOut {
+
+    public MockOfPacketOut() {
+        super(OFType.PACKET_OUT);
+    }
+
+    @Override
+    public OFBufferId getBufferId() {
+        return null;
+    }
+
+    @Override
+    public OFPort getInPort() {
+        return null;
+    }
+
+    @Override
+    public List<OFAction> getActions() {
+        return null;
+    }
+
+    @Override
+    public byte[] getData() {
+        return new byte[0];
+    }
+
+    @Override
+    public OFPacketOut.Builder createBuilder() {
+        return null;
+    }
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openflow.controller.driver;
+
+
+
+import org.projectfloodlight.openflow.protocol.OFStatsRequest;
+import org.projectfloodlight.openflow.protocol.OFStatsType;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFStatsReplyFlags;
+
+import java.util.Set;
+
+/**
+ * Mock of the Open Flow stats request message.
+ */
+public class MockOfStatsRequest extends OfMessageAdapter implements OFStatsRequest {
+
+    public MockOfStatsRequest() {
+        super(OFType.STATS_REQUEST);
+    }
+
+    @Override
+    public OFStatsType getStatsType() {
+        return null;
+    }
+
+    @Override
+    public Set<OFStatsReplyFlags> getFlags() {
+        return null;
+    }
+
+    @Override
+    public OFStatsRequest.Builder createBuilder() {
+        return null;
+    }
+}

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openflow.controller.driver;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.projectfloodlight.openflow.protocol.OFMessage;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.OFVersion;
+
+import com.google.common.hash.PrimitiveSink;
+
+/**
+ * Adapter for testing against an OpenFlow message.
+ */
+public class OfMessageAdapter implements OFMessage {
+    OFType type;
+
+    private OfMessageAdapter() {}
+
+    public OfMessageAdapter(OFType type) {
+        this.type = type;
+    }
+
+    @Override
+    public OFType getType() {
+        return type;
+    }
+
+    @Override
+    public OFVersion getVersion() {
+        return null;
+    }
+
+    @Override
+    public long getXid() {
+        return 0;
+    }
+
+    @Override
+    public void writeTo(ChannelBuffer channelBuffer) { }
+
+    @Override
+    public Builder createBuilder() {
+        return null;
+    }
+
+    @Override
+    public void putTo(PrimitiveSink sink) { }
+}

@@ -272,10 +272,13 @@ public class OpenFlowControllerImpl implements OpenFlowController {
             for (PacketListener p : ofPacketListener.values()) {
                 p.handlePacket(pktCtx);
             }
+            executorMsgs.submit(new OFMessageHandler(dpid, msg));
             break;
         // TODO: Consider using separate threadpool for sensitive messages.
         //    ie. Back to back error could cause us to starve.
         case FLOW_REMOVED:
+            executorMsgs.submit(new OFMessageHandler(dpid, msg));
+            break;
         case ERROR:
             executorMsgs.submit(new OFMessageHandler(dpid, msg));
             break;
@@ -625,6 +628,9 @@ public class OpenFlowControllerImpl implements OpenFlowController {
         }
     }
 
+    /**
+     * OpenFlow message handler for incoming control messages.
+     */
     protected final class OFMessageHandler implements Runnable {
 
         protected final OFMessage msg;
@@ -641,7 +647,5 @@ public class OpenFlowControllerImpl implements OpenFlowController {
                 listener.handleMessage(dpid, msg);
             }
         }
-
     }
-
 }

@@ -22,6 +22,7 @@ import org.onosproject.net.Device;
 import org.onosproject.net.driver.DriverData;
 import org.onosproject.net.driver.DriverHandler;
 import org.onosproject.openflow.controller.Dpid;
+import org.onosproject.openflow.controller.OpenFlowEventListener;
 import org.onosproject.openflow.controller.RoleState;
 import org.onosproject.openflow.controller.driver.OpenFlowAgent;
 import org.onosproject.openflow.controller.driver.OpenFlowSwitchDriver;
@@ -299,4 +300,12 @@ public class OpenflowSwitchDriverAdapter implements OpenFlowSwitchDriver {
     public String channelId() {
         return null;
     }
+
+    @Override
+    public void addEventListener(OpenFlowEventListener listener) {
+    }
+
+    @Override
+    public void removeEventListener(OpenFlowEventListener listener) {
+    }
 }

+/*
+ * Copyright 2015 Open Networking Laboratory
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.onosproject.openflow.controller.impl;
+
+import org.onosproject.openflow.OfMessageAdapter;
+import org.projectfloodlight.openflow.protocol.OFFlowRemoved;
+import org.projectfloodlight.openflow.protocol.OFType;
+import org.projectfloodlight.openflow.protocol.match.Match;
+import org.projectfloodlight.openflow.types.TableId;
+import org.projectfloodlight.openflow.types.U64;
+
+/**
+ * Mock of the Open Flow packet removed message.
+ */
+public class MockOfFlowRemoved extends OfMessageAdapter implements OFFlowRemoved {
+
+    public MockOfFlowRemoved() {
+        super(OFType.FLOW_REMOVED);
+    }
+
+    @Override
+    public U64 getCookie() {
+        return null;
+    }
+
+    @Override
+    public int getPriority() {
+        return 0;
+    }
+
+    @Override
+    public short getReason() {
+        return 0;
+    }
+
+    @Override
+    public TableId getTableId() throws UnsupportedOperationException {
+        return null;
+    }
+
+    @Override
+    public long getDurationSec() {
+        return 0;
+    }
+
+    @Override
+    public long getDurationNsec() {
+        return 0;
+    }
+
+    @Override
+    public int getIdleTimeout() {
+        return 0;
+    }
+
+    @Override
+    public int getHardTimeout() throws UnsupportedOperationException {
+        return 0;
+    }
+
+    @Override
+    public U64 getPacketCount() {
+        return null;
+    }
+
+    @Override
+    public U64 getByteCount() {
+        return null;
+    }
+
+    @Override
+    public Match getMatch() {
+        return null;
+    }
+
+    @Override
+    public OFFlowRemoved.Builder createBuilder() {
+        return null;
+    }
+}

@@ -24,12 +24,12 @@ import java.util.concurrent.Future;
 import org.junit.Before;
 import org.junit.Test;
 import org.onosproject.openflow.ExecutorServiceAdapter;
-import org.onosproject.openflow.MockOfFeaturesReply;
-import org.onosproject.openflow.MockOfPacketIn;
 import org.onosproject.openflow.MockOfPortStatus;
-import org.onosproject.openflow.OfMessageAdapter;
 import org.onosproject.openflow.OpenFlowSwitchListenerAdapter;
 import org.onosproject.openflow.OpenflowSwitchDriverAdapter;
+import org.onosproject.openflow.MockOfFeaturesReply;
+import org.onosproject.openflow.MockOfPacketIn;
+import org.onosproject.openflow.OfMessageAdapter;
 import org.onosproject.openflow.controller.Dpid;
 import org.onosproject.openflow.controller.OpenFlowPacketContext;
 import org.onosproject.openflow.controller.OpenFlowSwitch;
@@ -143,14 +143,16 @@ public class OpenFlowControllerImplPacketsTest {
     }
 
     /**
-     * Tests a packet in operation.
+     * Tests a packet in listen operation.
      */
     @Test
-    public void testPacketIn() {
+    public void testPacketInListen() {
         agent.addConnectedSwitch(dpid1, switch1);
         OFMessage packetInPacket = new MockOfPacketIn();
         controller.processPacket(dpid1, packetInPacket);
         assertThat(packetListener.contexts(), hasSize(1));
+        assertThat(executorService.submittedMessages(), hasSize(1));
+        assertThat(executorService.submittedMessages().get(0), is(packetInPacket));
     }
 
     /**
@@ -164,4 +166,16 @@ public class OpenFlowControllerImplPacketsTest {
         assertThat(executorService.submittedMessages(), hasSize(1));
         assertThat(executorService.submittedMessages().get(0), is(errorPacket));
     }
+
+    /**
+     * Tests a packet in operation.
+     */
+    @Test
+    public void testFlowRemoved() {
+        agent.addConnectedSwitch(dpid1, switch1);
+        OFMessage flowRemovedPacket = new MockOfFlowRemoved();
+        controller.processPacket(dpid1, flowRemovedPacket);
+        assertThat(executorService.submittedMessages(), hasSize(1));
+        assertThat(executorService.submittedMessages().get(0), is(flowRemovedPacket));
+    }
 }

@@ -140,6 +140,12 @@ public class OpenFlowDeviceProvider extends AbstractProvider implements DevicePr
 
     private final InternalDeviceProvider listener = new InternalDeviceProvider();
 
+    private final IncomingMessageProvider inMsgListener = new IncomingMessageProvider();
+
+    private final OutgoingMessageProvider outMsgListener = new OutgoingMessageProvider();
+
+    private boolean isCtrlMsgMonitor;
+
     // TODO: We need to make the poll interval configurable.
     static final int POLL_INTERVAL = 5;
     @Property(name = "PortStatsPollFrequency", intValue = POLL_INTERVAL,
@@ -161,6 +167,7 @@ public class OpenFlowDeviceProvider extends AbstractProvider implements DevicePr
         providerService = providerRegistry.register(this);
         controller.addListener(listener);
         controller.addEventListener(listener);
+
         connectInitialDevices();
         LOG.info("Started");
     }
@@ -264,6 +271,31 @@ public class OpenFlowDeviceProvider extends AbstractProvider implements DevicePr
         LOG.debug("Accepting mastership role change for device {}", deviceId);
     }
 
+
+    /**
+     * Enable OpenFlow control message monitoring.
+     */
+    public void enableCtrlMsgMonitor() {
+        isCtrlMsgMonitor = true;
+        controller.addEventListener(inMsgListener);
+        for (OpenFlowSwitch sw : controller.getSwitches()) {
+            sw.addEventListener(outMsgListener);
+        }
+        LOG.info("Enable control message monitoring.");
+    }
+
+    /**
+     * Disable OpenFlow control message monitoring.
+     */
+    public void disableCtrlMsgMonitor() {
+        isCtrlMsgMonitor = false;
+        controller.removeEventListener(inMsgListener);
+        for (OpenFlowSwitch sw: controller.getSwitches()) {
+            sw.removeEventListener(outMsgListener);
+        }
+        LOG.info("Disable control message monitoring");
+    }
+
     private void pushPortMetrics(Dpid dpid, List<OFPortStatsEntry> portStatsEntries) {
         DeviceId deviceId = DeviceId.deviceId(dpid.uri(dpid));
         Collection<PortStatistics> stats = buildPortStatistics(deviceId, portStatsEntries);
@@ -304,6 +336,32 @@ public class OpenFlowDeviceProvider extends AbstractProvider implements DevicePr
 
     }
 
+    /**
+     * A listener for incoming OpenFlow messages.
+     */
+    private class IncomingMessageProvider implements OpenFlowEventListener {
+
+        @Override
+        public void handleMessage(Dpid dpid, OFMessage msg) {
+            if (isCtrlMsgMonitor) {
+                // TODO: feed the control message stats via ControlMetricsServiceFactory
+            }
+        }
+    }
+
+    /**
+     * A listener for outgoing OpenFlow messages.
+     */
+    private class OutgoingMessageProvider implements OpenFlowEventListener {
+
+        @Override
+        public void handleMessage(Dpid dpid, OFMessage msg) {
+            if (isCtrlMsgMonitor) {
+                // TODO: feed the control message stats via ControlMetricsServiceFactory
+            }
+        }
+    }
+
     private class InternalDeviceProvider implements OpenFlowSwitchListener, OpenFlowEventListener {
 
         private HashMap<Dpid, List<OFPortStatsEntry>> portStatsReplies = new HashMap<>();
@@ -319,6 +377,11 @@ public class OpenFlowDeviceProvider extends AbstractProvider implements DevicePr
                 return;
             }
 
+            if (isCtrlMsgMonitor) {
+                // start to monitor the outgoing control messages
+                sw.addEventListener(outMsgListener);
+            }
+
             ChassisId cId = new ChassisId(dpid.value());
 
             SparseAnnotations annotations = DefaultAnnotations.builder()
@@ -359,6 +422,14 @@ public class OpenFlowDeviceProvider extends AbstractProvider implements DevicePr
             if (collector != null) {
                 collector.stop();
             }
+
+            OpenFlowSwitch sw = controller.getSwitch(dpid);
+            if (sw != null) {
+                if (isCtrlMsgMonitor) {
+                    // stop monitoring the outgoing control messages
+                    sw.removeEventListener(outMsgListener);
+                }
+            }
         }
 
         @Override

@@ -398,6 +398,14 @@ public class OpenFlowDeviceProviderTest {
             return "1.2.3.4:1";
         }
 
+        @Override
+        public void addEventListener(OpenFlowEventListener listener) {
+        }
+
+        @Override
+        public void removeEventListener(OpenFlowEventListener listener) {
+        }
+
     }
 
 }

@@ -408,5 +408,13 @@ public class OpenFlowGroupProviderTest {
             return null;
         }
 
+        @Override
+        public void addEventListener(OpenFlowEventListener listener) {
+        }
+
+        @Override
+        public void removeEventListener(OpenFlowEventListener listener) {
+        }
+
     }
 }
\ No newline at end of file

@@ -425,6 +425,14 @@ public class OpenFlowPacketProviderTest {
             return "1.2.3.4:1";
         }
 
+        @Override
+        public void addEventListener(OpenFlowEventListener listener) {
+        }
+
+        @Override
+        public void removeEventListener(OpenFlowEventListener listener) {
+        }
+
 
     }