New flowrule store that utilizes a simple mechanism to periodically backsup its flowentries.

For efficiency reasons backups are only run for those devices whose flow entries are updated (since last backup) or if the device master/backup has changed.
This backup mechanism will be a short term solution until we get to a more close to real-time backup approach.
This change also disables the flowrule store based on hazelcast.

Change-Id: Iaae08852edee20b999ff97c60ca8bc6576e645f6

@@ -17,8 +17,8 @@ package org.onosproject.store.flow;
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
-import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 
 import org.onosproject.cluster.NodeId;
 
@@ -30,15 +30,15 @@ import com.google.common.base.Optional;
 public final class ReplicaInfo {
 
     private final Optional<NodeId> master;
-    private final Collection<NodeId> backups;
+    private final List<NodeId> backups;
 
     /**
      * Creates a ReplicaInfo instance.
      *
      * @param master NodeId of the node where the master copy should be
-     * @param backups collection of NodeId, where backup copies should be placed
+     * @param backups list of NodeId, where backup copies should be placed
      */
-    public ReplicaInfo(NodeId master, Collection<NodeId> backups) {
+    public ReplicaInfo(NodeId master, List<NodeId> backups) {
         this.master = Optional.fromNullable(master);
         this.backups = checkNotNull(backups);
     }
@@ -57,7 +57,7 @@ public final class ReplicaInfo {
      *
      * @return collection of NodeId, where backup copies should be placed
      */
-    public Collection<NodeId> backups() {
+    public List<NodeId> backups() {
         return backups;
     }
 

@@ -108,7 +108,7 @@ import static org.slf4j.LoggerFactory.getLogger;
 /**
  * Manages inventory of flow rules using a distributed state management protocol.
  */
-@Component(immediate = true)
+@Component(immediate = false, enabled = false)
 @Service
 public class DistributedFlowRuleStore
         extends AbstractHazelcastStore<FlowRuleBatchEvent, FlowRuleStoreDelegate>

@@ -36,5 +36,8 @@ public final class FlowStoreMessageSubjects {
         = new MessageSubject("peer-forward-remove-flow-entry");
 
     public static final MessageSubject REMOTE_APPLY_COMPLETED
-            = new MessageSubject("peer-apply-completed");
+        = new MessageSubject("peer-apply-completed");
+
+    public static final MessageSubject FLOW_TABLE_BACKUP
+        = new MessageSubject("peer-flow-table-backup");
 }

@@ -21,6 +21,8 @@ import static org.onosproject.store.flow.ReplicaInfoEvent.Type.MASTER_CHANGED;
 import static org.onosproject.store.flow.ReplicaInfoEvent.Type.BACKUPS_CHANGED;
 
 import java.util.Collections;
+import java.util.List;
+
 import org.apache.felix.scr.annotations.Activate;
 import org.apache.felix.scr.annotations.Component;
 import org.apache.felix.scr.annotations.Deactivate;
@@ -28,6 +30,7 @@ import org.apache.felix.scr.annotations.Reference;
 import org.apache.felix.scr.annotations.ReferenceCardinality;
 import org.apache.felix.scr.annotations.Service;
 import org.onosproject.cluster.NodeId;
+import org.onosproject.cluster.RoleInfo;
 import org.onosproject.event.AbstractListenerRegistry;
 import org.onosproject.event.EventDeliveryService;
 import org.onosproject.mastership.MastershipEvent;
@@ -76,9 +79,7 @@ public class ReplicaInfoManager implements ReplicaInfoService {
 
     @Override
     public ReplicaInfo getReplicaInfoFor(DeviceId deviceId) {
-        // TODO: populate backup List when we reach the point we need them.
-        return new ReplicaInfo(mastershipService.getMasterFor(deviceId),
-                               Collections.<NodeId>emptyList());
+        return buildFromRoleInfo(mastershipService.getNodesFor(deviceId));
     }
 
     @Override
@@ -91,13 +92,17 @@ public class ReplicaInfoManager implements ReplicaInfoService {
         listenerRegistry.removeListener(checkNotNull(listener));
     }
 
+    private static ReplicaInfo buildFromRoleInfo(RoleInfo roles) {
+        List<NodeId> backups = roles.backups() == null ?
+                Collections.emptyList() : roles.backups();
+        return new ReplicaInfo(roles.master(), backups);
+    }
+
     final class InternalMastershipListener implements MastershipListener {
 
         @Override
         public void event(MastershipEvent event) {
-            final ReplicaInfo replicaInfo
-                = new ReplicaInfo(event.roleInfo().master(),
-                                  event.roleInfo().backups());
+            final ReplicaInfo replicaInfo = buildFromRoleInfo(event.roleInfo());
 
             switch (event.type()) {
             case MASTER_CHANGED:

@@ -38,7 +38,7 @@ import java.io.InputStream;
 /**
  * Auxiliary bootstrap of distributed store.
  */
-@Component(immediate = true)
+@Component(immediate = false, enabled = false)
 @Service
 public class StoreManager implements StoreService {
 

@@ -218,6 +218,8 @@ public class ConsistentDeviceMastershipStore
         NodeId master = null;
         final List<NodeId> standbys = Lists.newLinkedList();
 
+        List<NodeId> candidates = leadershipService.getCandidates(createDeviceMastershipTopic(deviceId));
+
         for (Map.Entry<NodeId, MastershipRole> entry : roles.entrySet()) {
             if (entry.getValue() == MastershipRole.MASTER) {
                 master = entry.getKey();
@@ -226,7 +228,9 @@ public class ConsistentDeviceMastershipStore
             }
         }
 
-        return new RoleInfo(master, standbys);
+        List<NodeId> sortedStandbyList = candidates.stream().filter(standbys::contains).collect(Collectors.toList());
+
+        return new RoleInfo(master, sortedStandbyList);
     }
 
     @Override

@@ -153,6 +153,11 @@ public class ReplicaInfoManagerTest {
         }
 
         @Override
+        public RoleInfo getNodesFor(DeviceId deviceId) {
+            return new RoleInfo(masters.get(deviceId), Collections.emptyList());
+        }
+
+        @Override
         public void addListener(MastershipListener listener) {
             mastershipListenerRegistry.addListener(listener);
         }

@@ -18,6 +18,7 @@ package org.onosproject.store.serializers;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
+
 import org.onlab.packet.ChassisId;
 import org.onlab.packet.Ip4Address;
 import org.onlab.packet.Ip4Prefix;
@@ -170,6 +171,8 @@ import java.util.HashMap;
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CopyOnWriteArraySet;
 
 public final class KryoNamespaces {
 
@@ -191,6 +194,8 @@ public final class KryoNamespaces {
                       ImmutableMap.of("a", 1).getClass(),
                       ImmutableMap.of("R", 2, "D", 2).getClass())
             .register(HashMap.class)
+            .register(ConcurrentHashMap.class)
+            .register(CopyOnWriteArraySet.class)
             .register(ArrayList.class,
                       LinkedList.class,
                       HashSet.class