Initial DistributedDlowRuleStore

- known bug: responding to ClusterMessage not possible.

Change-Id: Iaa4245c64d2a6219d7c48ed30ddca7d558dbc177

@@ -24,7 +24,18 @@ public class FlowRuleEvent extends AbstractEvent<FlowRuleEvent.Type, FlowRule> {
         /**
         /**
          * Signifies that a rule has been updated.
          * Signifies that a rule has been updated.
          */
          */
-        RULE_UPDATED
+        RULE_UPDATED,
+
+        // internal event between Manager <-> Store
+
+        /*
+         * Signifies that a request to add flow rule has been added to the store.
+         */
+        RULE_ADD_REQUESTED,
+        /*
+         * Signifies that a request to remove flow rule has been added to the store.
+         */
+        RULE_REMOVE_REQUESTED,
     }
     }
 
 
     /**
     /**

@@ -44,16 +44,18 @@ public interface FlowRuleStore extends Store<FlowRuleEvent, FlowRuleStoreDelegat
      * Stores a new flow rule without generating events.
      * Stores a new flow rule without generating events.
      *
      *
      * @param rule the flow rule to add
      * @param rule the flow rule to add
+     * @return true if the rule should be handled locally
      */
      */
-    void storeFlowRule(FlowRule rule);
+    boolean storeFlowRule(FlowRule rule);
 
 
     /**
     /**
      * Marks a flow rule for deletion. Actual deletion will occur
      * Marks a flow rule for deletion. Actual deletion will occur
      * when the provider indicates that the flow has been removed.
      * when the provider indicates that the flow has been removed.
      *
      *
      * @param rule the flow rule to delete
      * @param rule the flow rule to delete
+     * @return true if the rule should be handled locally
      */
      */
-    void deleteFlowRule(FlowRule rule);
+    boolean deleteFlowRule(FlowRule rule);
 
 
     /**
     /**
      * Stores a new flow rule, or updates an existing entry.
      * Stores a new flow rule, or updates an existing entry.

@@ -104,24 +104,52 @@ public class FlowRuleManager
     public void applyFlowRules(FlowRule... flowRules) {
     public void applyFlowRules(FlowRule... flowRules) {
         for (int i = 0; i < flowRules.length; i++) {
         for (int i = 0; i < flowRules.length; i++) {
             FlowRule f = flowRules[i];
             FlowRule f = flowRules[i];
-            final Device device = deviceService.getDevice(f.deviceId());
-            final FlowRuleProvider frp = getProvider(device.providerId());
-            store.storeFlowRule(f);
-            frp.applyFlowRule(f);
+            boolean local = store.storeFlowRule(f);
+            if (local) {
+                // TODO: aggregate all local rules and push down once?
+                applyFlowRulesToProviders(f);
+            }
+        }
+    }
+
+    private void applyFlowRulesToProviders(FlowRule... flowRules) {
+        DeviceId did = null;
+        FlowRuleProvider frp = null;
+        for (FlowRule f : flowRules) {
+            if (!f.deviceId().equals(did)) {
+                did = f.deviceId();
+                final Device device = deviceService.getDevice(did);
+                frp = getProvider(device.providerId());
+            }
+            if (frp != null) {
+                frp.applyFlowRule(f);
+            }
         }
         }
     }
     }
 
 
     @Override
     @Override
     public void removeFlowRules(FlowRule... flowRules) {
     public void removeFlowRules(FlowRule... flowRules) {
         FlowRule f;
         FlowRule f;
-        FlowRuleProvider frp;
-        Device device;
         for (int i = 0; i < flowRules.length; i++) {
         for (int i = 0; i < flowRules.length; i++) {
             f = flowRules[i];
             f = flowRules[i];
-            device = deviceService.getDevice(f.deviceId());
-            store.deleteFlowRule(f);
-            if (device != null) {
+            boolean local = store.deleteFlowRule(f);
+            if (local) {
+                // TODO: aggregate all local rules and push down once?
+                removeFlowRulesFromProviders(f);
+            }
+        }
+    }
+
+    private void removeFlowRulesFromProviders(FlowRule... flowRules) {
+        DeviceId did = null;
+        FlowRuleProvider frp = null;
+        for (FlowRule f : flowRules) {
+            if (!f.deviceId().equals(did)) {
+                did = f.deviceId();
+                final Device device = deviceService.getDevice(did);
                 frp = getProvider(device.providerId());
                 frp = getProvider(device.providerId());
+            }
+            if (frp != null) {
                 frp.removeFlowRule(f);
                 frp.removeFlowRule(f);
             }
             }
         }
         }
@@ -135,8 +163,11 @@ public class FlowRuleManager
 
 
         for (FlowRule f : rules) {
         for (FlowRule f : rules) {
             store.deleteFlowRule(f);
             store.deleteFlowRule(f);
+            // FIXME: only accept request and push to provider on internal event
             device = deviceService.getDevice(f.deviceId());
             device = deviceService.getDevice(f.deviceId());
             frp = getProvider(device.providerId());
             frp = getProvider(device.providerId());
+            // FIXME: flows removed from store and flows removed from might diverge
+            //        get rid of #removeRulesById?
             frp.removeRulesById(id, f);
             frp.removeRulesById(id, f);
         }
         }
     }
     }
@@ -352,7 +383,23 @@ public class FlowRuleManager
     private class InternalStoreDelegate implements FlowRuleStoreDelegate {
     private class InternalStoreDelegate implements FlowRuleStoreDelegate {
         @Override
         @Override
         public void notify(FlowRuleEvent event) {
         public void notify(FlowRuleEvent event) {
-            eventDispatcher.post(event);
+            switch (event.type()) {
+            case RULE_ADD_REQUESTED:
+                applyFlowRulesToProviders(event.subject());
+                break;
+            case RULE_REMOVE_REQUESTED:
+                removeFlowRulesFromProviders(event.subject());
+                break;
+
+            case RULE_ADDED:
+            case RULE_REMOVED:
+            case RULE_UPDATED:
+                // only dispatch events related to switch
+                eventDispatcher.post(event);
+                break;
+            default:
+                break;
+            }
         }
         }
     }
     }
 
 

@@ -2,6 +2,7 @@ package org.onlab.onos.store.flow.impl;
 
 
 import static org.onlab.onos.net.flow.FlowRuleEvent.Type.RULE_REMOVED;
 import static org.onlab.onos.net.flow.FlowRuleEvent.Type.RULE_REMOVED;
 import static org.slf4j.LoggerFactory.getLogger;
 import static org.slf4j.LoggerFactory.getLogger;
+import static org.onlab.onos.store.flow.impl.FlowStoreMessageSubjects.*;
 
 
 import java.io.IOException;
 import java.io.IOException;
 import java.util.Collection;
 import java.util.Collection;
@@ -30,6 +31,7 @@ import org.onlab.onos.net.flow.StoredFlowEntry;
 import org.onlab.onos.store.AbstractStore;
 import org.onlab.onos.store.AbstractStore;
 import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
 import org.onlab.onos.store.cluster.messaging.ClusterCommunicationService;
 import org.onlab.onos.store.cluster.messaging.ClusterMessage;
 import org.onlab.onos.store.cluster.messaging.ClusterMessage;
+import org.onlab.onos.store.cluster.messaging.ClusterMessageHandler;
 import org.onlab.onos.store.cluster.messaging.ClusterMessageResponse;
 import org.onlab.onos.store.cluster.messaging.ClusterMessageResponse;
 import org.onlab.onos.store.flow.ReplicaInfo;
 import org.onlab.onos.store.flow.ReplicaInfo;
 import org.onlab.onos.store.flow.ReplicaInfoService;
 import org.onlab.onos.store.flow.ReplicaInfoService;
@@ -80,10 +82,44 @@ public class DistributedFlowRuleStore
     };
     };
 
 
     // TODO: make this configurable
     // TODO: make this configurable
-    private static final long FLOW_RULE_STORE_TIMEOUT_MILLIS = 1000;
+    private static final long FLOW_RULE_STORE_TIMEOUT_MILLIS = 5000;
 
 
     @Activate
     @Activate
     public void activate() {
     public void activate() {
+        clusterCommunicator.addSubscriber(STORE_FLOW_RULE, new ClusterMessageHandler() {
+
+            @Override
+            public void handle(ClusterMessage message) {
+                FlowRule rule = SERIALIZER.decode(message.payload());
+                log.info("received add request for {}", rule);
+                storeFlowEntryInternal(rule);
+                // FIXME what to respond.
+                try {
+                    // FIXME: #respond() not working. responded message is
+                    // handled by this sender node and never goes back.
+                    message.respond(SERIALIZER.encode("ACK"));
+                } catch (IOException e) {
+                    log.error("Failed to respond back", e);
+                }
+            }
+        });
+
+        clusterCommunicator.addSubscriber(DELETE_FLOW_RULE, new ClusterMessageHandler() {
+
+            @Override
+            public void handle(ClusterMessage message) {
+                FlowRule rule = SERIALIZER.decode(message.payload());
+                log.info("received delete request for {}", rule);
+                deleteFlowRuleInternal(rule);
+                // FIXME what to respond.
+                try {
+                    message.respond(SERIALIZER.encode("ACK"));
+                } catch (IOException e) {
+                    log.error("Failed to respond back", e);
+                }
+
+            }
+        });
         log.info("Started");
         log.info("Started");
     }
     }
 
 
@@ -131,13 +167,14 @@ public class DistributedFlowRuleStore
     }
     }
 
 
     @Override
     @Override
-    public void storeFlowRule(FlowRule rule) {
+    public boolean storeFlowRule(FlowRule rule) {
         ReplicaInfo replicaInfo = replicaInfoManager.getReplicaInfoFor(rule.deviceId());
         ReplicaInfo replicaInfo = replicaInfoManager.getReplicaInfoFor(rule.deviceId());
         if (replicaInfo.master().get().equals(clusterService.getLocalNode().id())) {
         if (replicaInfo.master().get().equals(clusterService.getLocalNode().id())) {
-            storeFlowEntryInternal(rule);
-            return;
+            return storeFlowEntryInternal(rule);
         }
         }
 
 
+        log.warn("Not my flow forwarding to {}", replicaInfo.master().orNull());
+
         ClusterMessage message = new ClusterMessage(
         ClusterMessage message = new ClusterMessage(
                 clusterService.getLocalNode().id(),
                 clusterService.getLocalNode().id(),
                 FlowStoreMessageSubjects.STORE_FLOW_RULE,
                 FlowStoreMessageSubjects.STORE_FLOW_RULE,
@@ -150,26 +187,29 @@ public class DistributedFlowRuleStore
             // FIXME: throw a FlowStoreException
             // FIXME: throw a FlowStoreException
             throw new RuntimeException(e);
             throw new RuntimeException(e);
         }
         }
+        return false;
     }
     }
 
 
-    private synchronized void storeFlowEntryInternal(FlowRule flowRule) {
+    private synchronized boolean storeFlowEntryInternal(FlowRule flowRule) {
         StoredFlowEntry flowEntry = new DefaultFlowEntry(flowRule);
         StoredFlowEntry flowEntry = new DefaultFlowEntry(flowRule);
         DeviceId deviceId = flowRule.deviceId();
         DeviceId deviceId = flowRule.deviceId();
         // write to local copy.
         // write to local copy.
         if (!flowEntries.containsEntry(deviceId, flowEntry)) {
         if (!flowEntries.containsEntry(deviceId, flowEntry)) {
             flowEntries.put(deviceId, flowEntry);
             flowEntries.put(deviceId, flowEntry);
             flowEntriesById.put(flowRule.appId(), flowEntry);
             flowEntriesById.put(flowRule.appId(), flowEntry);
+            notifyDelegate(new FlowRuleEvent(Type.RULE_ADD_REQUESTED, flowRule));
+            return true;
         }
         }
         // write to backup.
         // write to backup.
         // TODO: write to a hazelcast map.
         // TODO: write to a hazelcast map.
+        return false;
     }
     }
 
 
     @Override
     @Override
-    public synchronized void deleteFlowRule(FlowRule rule) {
+    public synchronized boolean deleteFlowRule(FlowRule rule) {
         ReplicaInfo replicaInfo = replicaInfoManager.getReplicaInfoFor(rule.deviceId());
         ReplicaInfo replicaInfo = replicaInfoManager.getReplicaInfoFor(rule.deviceId());
         if (replicaInfo.master().get().equals(clusterService.getLocalNode().id())) {
         if (replicaInfo.master().get().equals(clusterService.getLocalNode().id())) {
-            deleteFlowRuleInternal(rule);
-            return;
+            return deleteFlowRuleInternal(rule);
         }
         }
 
 
         ClusterMessage message = new ClusterMessage(
         ClusterMessage message = new ClusterMessage(
@@ -184,15 +224,21 @@ public class DistributedFlowRuleStore
             // FIXME: throw a FlowStoreException
             // FIXME: throw a FlowStoreException
             throw new RuntimeException(e);
             throw new RuntimeException(e);
         }
         }
+        return false;
     }
     }
 
 
-    private synchronized void deleteFlowRuleInternal(FlowRule flowRule) {
+    private synchronized boolean deleteFlowRuleInternal(FlowRule flowRule) {
         StoredFlowEntry entry = getFlowEntryInternal(flowRule);
         StoredFlowEntry entry = getFlowEntryInternal(flowRule);
         if (entry == null) {
         if (entry == null) {
-            return;
+            return false;
         }
         }
         entry.setState(FlowEntryState.PENDING_REMOVE);
         entry.setState(FlowEntryState.PENDING_REMOVE);
+
         // TODO: also update backup.
         // TODO: also update backup.
+
+        notifyDelegate(new FlowRuleEvent(Type.RULE_REMOVE_REQUESTED, flowRule));
+
+        return true;
     }
     }
 
 
     @Override
     @Override

@@ -148,8 +148,9 @@ public class SimpleFlowRuleStore
     }
     }
 
 
     @Override
     @Override
-    public void storeFlowRule(FlowRule rule) {
+    public boolean storeFlowRule(FlowRule rule) {
         final boolean added = storeFlowRuleInternal(rule);
         final boolean added = storeFlowRuleInternal(rule);
+        return added;
     }
     }
 
 
     private boolean storeFlowRuleInternal(FlowRule rule) {
     private boolean storeFlowRuleInternal(FlowRule rule) {
@@ -166,13 +167,14 @@ public class SimpleFlowRuleStore
             }
             }
             // new flow rule added
             // new flow rule added
             existing.add(f);
             existing.add(f);
-            // TODO: notify through delegate about remote event?
+            // TODO: Should we notify only if it's "remote" event?
+            //notifyDelegate(new FlowRuleEvent(Type.RULE_ADD_REQUESTED, rule));
             return true;
             return true;
         }
         }
     }
     }
 
 
     @Override
     @Override
-    public void deleteFlowRule(FlowRule rule) {
+    public boolean deleteFlowRule(FlowRule rule) {
 
 
         List<StoredFlowEntry> entries = getFlowEntries(rule.deviceId(), rule.id());
         List<StoredFlowEntry> entries = getFlowEntries(rule.deviceId(), rule.id());
         synchronized (entries) {
         synchronized (entries) {
@@ -180,12 +182,15 @@ public class SimpleFlowRuleStore
                 if (entry.equals(rule)) {
                 if (entry.equals(rule)) {
                     synchronized (entry) {
                     synchronized (entry) {
                         entry.setState(FlowEntryState.PENDING_REMOVE);
                         entry.setState(FlowEntryState.PENDING_REMOVE);
-                        return;
+                        // TODO: Should we notify only if it's "remote" event?
+                        //notifyDelegate(new FlowRuleEvent(Type.RULE_REMOVE_REQUESTED, rule));
+                        return true;
                     }
                     }
                 }
                 }
             }
             }
         }
         }
         //log.warn("Cannot find rule {}", rule);
         //log.warn("Cannot find rule {}", rule);
+        return false;
     }
     }
 
 
     @Override
     @Override