CORD-48 Implementation of hashing Next Objective in OF-DPA driver. Major changes…

… to ensure multi-ONOS-instance group-chain installation.
Also includes:
     Changes to Next Objective that adds metadata field for applications to optionally send auxillary info to drivers
     Changes to Next Objective that allows more explicit modification of the next objective
     Changes to Forwarding Objective and PendingNext to include hashCode() and equals() method
     MplsBosInstruction included in kryo serializer
     GroupKey's byte[] represented as a hex string
     Bug fix in mpls flow installation to report failure in install
     Bug fix in linkUp in SR app to disallow non-masters to modify groups
     Bug fix in ordering of actions in group

Change-Id: I3e7003f55724c2de79589e43e11d05ff4815a81d

@@ -448,7 +448,6 @@ public class DefaultRoutingHandler {
         if (nextHops.isEmpty()) {
             nextHops.add(destSw);
         }
-
         // If both target switch and dest switch are edge routers, then set IP
         // rule for both subnet and router IP.
         boolean targetIsEdge;
@@ -467,7 +466,7 @@ public class DefaultRoutingHandler {
         if (targetIsEdge && destIsEdge) {
             Set<Ip4Prefix> subnets = config.getSubnets(destSw);
             log.debug("* populateEcmpRoutingRulePartial in device {} towards {} for subnets {}",
-                    targetSw, destSw, subnets);
+                      targetSw, destSw, subnets);
             result = rulePopulator.populateIpRuleForSubnet(targetSw,
                                                            subnets,
                                                            destSw,
@@ -479,24 +478,23 @@ public class DefaultRoutingHandler {
             Ip4Address routerIp = destRouterIp;
             IpPrefix routerIpPrefix = IpPrefix.valueOf(routerIp, IpPrefix.MAX_INET_MASK_LENGTH);
             log.debug("* populateEcmpRoutingRulePartial in device {} towards {} for router IP {}",
-                    targetSw, destSw, routerIpPrefix);
+                      targetSw, destSw, routerIpPrefix);
             result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix, destSw, nextHops);
             if (!result) {
                 return false;
             }
 
-        // If the target switch is an edge router, then set IP rules for the router IP.
         } else if (targetIsEdge) {
+            // If the target switch is an edge router, then set IP rules for the router IP.
             Ip4Address routerIp = destRouterIp;
             IpPrefix routerIpPrefix = IpPrefix.valueOf(routerIp, IpPrefix.MAX_INET_MASK_LENGTH);
             log.debug("* populateEcmpRoutingRulePartial in device {} towards {} for router IP {}",
-                    targetSw, destSw, routerIpPrefix);
+                      targetSw, destSw, routerIpPrefix);
             result = rulePopulator.populateIpRuleForRouter(targetSw, routerIpPrefix, destSw, nextHops);
             if (!result) {
                 return false;
             }
         }
-
         // Populates MPLS rules to all routers
         log.debug("* populateEcmpRoutingRulePartial in device{} towards {} for all MPLS rules",
                 targetSw, destSw);
@@ -504,7 +502,6 @@ public class DefaultRoutingHandler {
         if (!result) {
             return false;
         }
-
         return true;
     }
 

@@ -424,17 +424,22 @@ public class SegmentRoutingManager implements SegmentRoutingService {
 
     /**
      * Returns the next objective ID for the given NeighborSet.
-     * If the nextObjectiveID does not exist, a new one is created and returned.
+     * If the nextObjective does not exist, a new one is created and
+     * it's id is returned.
+     * TODO move the side-effect creation of a Next Objective into a new method
      *
      * @param deviceId Device ID
      * @param ns NegighborSet
-     * @return next objective ID
+     * @param meta metadata passed into the creation of a Next Objective
+     * @return next objective ID or -1 if an error was encountered during the
+     *         creation of the nextObjective
      */
-    public int getNextObjectiveId(DeviceId deviceId, NeighborSet ns) {
+    public int getNextObjectiveId(DeviceId deviceId, NeighborSet ns,
+                                  TrafficSelector meta) {
         if (groupHandlerMap.get(deviceId) != null) {
             log.trace("getNextObjectiveId query in device {}", deviceId);
             return groupHandlerMap
-                    .get(deviceId).getNextObjectiveId(ns);
+                    .get(deviceId).getNextObjectiveId(ns, meta);
         } else {
             log.warn("getNextObjectiveId query in device {} not found", deviceId);
             return -1;
@@ -586,7 +591,8 @@ public class SegmentRoutingManager implements SegmentRoutingService {
         DefaultGroupHandler groupHandler = groupHandlerMap.get(link.src()
                 .deviceId());
         if (groupHandler != null) {
-            groupHandler.linkUp(link);
+            groupHandler.linkUp(link, mastershipService.isLocalMaster(
+                                           link.src().deviceId()));
         } else {
             Device device = deviceService.getDevice(link.src().deviceId());
             if (device != null) {
@@ -596,7 +602,7 @@ public class SegmentRoutingManager implements SegmentRoutingService {
                 processDeviceAdded(device);
                 groupHandler = groupHandlerMap.get(link.src()
                                                    .deviceId());
-                groupHandler.linkUp(link);
+                groupHandler.linkUp(link, mastershipService.isLocalMaster(device.id()));
             }
         }
 

@@ -194,7 +194,7 @@ public class TunnelHandler {
             tunnel.allowToRemoveGroup(true);
         }
 
-        return groupHandlerMap.get(deviceId).getNextObjectiveId(ns);
+        return groupHandlerMap.get(deviceId).getNextObjectiveId(ns, null);
     }
 
 }

@@ -93,7 +93,7 @@ public class DefaultEdgeGroupHandler extends DefaultGroupHandler {
                 + "with label for sw {} is {}",
                 deviceId, nsSet);
 
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
 
     @Override
@@ -107,7 +107,7 @@ public class DefaultEdgeGroupHandler extends DefaultGroupHandler {
         Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(
                                              newNeighborLink.dst().deviceId(),
                                              devicePortMap.keySet());
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
 
     @Override

@@ -81,7 +81,7 @@ public class DefaultTransitGroupHandler extends DefaultGroupHandler {
         log.debug("createGroupsAtTransitRouter: The neighborset with label "
                 + "for sw {} is {}", deviceId, nsSet);
 
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
 
     @Override
@@ -95,7 +95,7 @@ public class DefaultTransitGroupHandler extends DefaultGroupHandler {
         Set<NeighborSet> nsSet = computeImpactedNeighborsetForPortEvent(
                                              newNeighborLink.dst().deviceId(),
                                              devicePortMap.keySet());
-        createGroupsFromNeighborsets(nsSet);
+        //createGroupsFromNeighborsets(nsSet);
     }
 
     @Override

@@ -16,6 +16,7 @@
 package org.onosproject.net.flowobjective;
 
 import com.google.common.annotations.Beta;
+
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
@@ -119,6 +120,53 @@ public final class DefaultForwardingObjective implements ForwardingObjective {
         return context;
     }
 
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#hashCode()
+     */
+    @Override
+    public int hashCode() {
+        return Objects.hash(selector, flag, permanent,
+                            timeout, appId, priority, nextId,
+                            treatment, op);
+    }
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see java.lang.Object#equals(java.lang.Object)
+     */
+    @Override
+    public boolean equals(final Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (!(obj instanceof DefaultForwardingObjective)) {
+            return false;
+        }
+        final DefaultForwardingObjective other = (DefaultForwardingObjective) obj;
+        boolean nextEq = false, treatmentEq = false;
+        if (this.selector.equals(other.selector) &&
+                this.flag == other.flag &&
+                this.permanent == other.permanent &&
+                this.timeout == other.timeout &&
+                this.appId.equals(other.appId) &&
+                this.priority == other.priority &&
+                this.op == other.op) {
+            if (this.nextId != null && other.nextId != null) {
+                nextEq = this.nextId == other.nextId;
+            }
+            if (this.treatment != null && other.treatment != null) {
+                treatmentEq = this.treatment.equals(other.treatment);
+            }
+            if (nextEq && treatmentEq) {
+                return true;
+            }
+        }
+        return false;
+    }
+
     /**
      * Returns a new builder.
      *

@@ -18,6 +18,7 @@ package org.onosproject.net.flowobjective;
 import com.google.common.annotations.Beta;
 import com.google.common.collect.ImmutableList;
 import org.onosproject.core.ApplicationId;
+import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 
 import java.util.Collection;
@@ -39,6 +40,7 @@ public final class DefaultNextObjective implements NextObjective {
     private final Integer id;
     private final Operation op;
     private final Optional<ObjectiveContext> context;
+    private final TrafficSelector meta;
 
     private DefaultNextObjective(Builder builder) {
         this.treatments = builder.treatments;
@@ -47,6 +49,7 @@ public final class DefaultNextObjective implements NextObjective {
         this.id = builder.id;
         this.op = builder.op;
         this.context = Optional.ofNullable(builder.context);
+        this.meta = builder.meta;
     }
 
     @Override
@@ -94,6 +97,11 @@ public final class DefaultNextObjective implements NextObjective {
         return context;
     }
 
+    @Override
+    public TrafficSelector meta() {
+        return meta;
+    }
+
     /**
      * Returns a new builder.
      *
@@ -111,6 +119,7 @@ public final class DefaultNextObjective implements NextObjective {
         private List<TrafficTreatment> treatments;
         private Operation op;
         private ObjectiveContext context;
+        private TrafficSelector meta;
 
         private final ImmutableList.Builder<TrafficTreatment> listBuilder
                 = ImmutableList.builder();
@@ -172,6 +181,12 @@ public final class DefaultNextObjective implements NextObjective {
         }
 
         @Override
+        public Builder setMeta(TrafficSelector meta) {
+            this.meta = meta;
+            return this;
+        }
+
+        @Override
         public NextObjective add() {
             treatments = listBuilder.build();
             op = Operation.ADD;
@@ -218,5 +233,55 @@ public final class DefaultNextObjective implements NextObjective {
 
             return new DefaultNextObjective(this);
         }
+
+        @Override
+        public NextObjective addToExisting() {
+            treatments = listBuilder.build();
+            op = Operation.ADD_TO_EXISTING;
+            checkNotNull(appId, "Must supply an application id");
+            checkNotNull(id, "id cannot be null");
+            checkNotNull(type, "The type cannot be null");
+            checkArgument(!treatments.isEmpty(), "Must have at least one treatment");
+
+            return new DefaultNextObjective(this);
+        }
+
+        @Override
+        public NextObjective removeFromExisting() {
+            treatments = listBuilder.build();
+            op = Operation.REMOVE_FROM_EXISTING;
+            checkNotNull(appId, "Must supply an application id");
+            checkNotNull(id, "id cannot be null");
+            checkNotNull(type, "The type cannot be null");
+
+            return new DefaultNextObjective(this);
+        }
+
+        @Override
+        public NextObjective addToExisting(ObjectiveContext context) {
+            treatments = listBuilder.build();
+            op = Operation.ADD_TO_EXISTING;
+            this.context = context;
+            checkNotNull(appId, "Must supply an application id");
+            checkNotNull(id, "id cannot be null");
+            checkNotNull(type, "The type cannot be null");
+            checkArgument(!treatments.isEmpty(), "Must have at least one treatment");
+
+            return new DefaultNextObjective(this);
+        }
+
+        @Override
+        public NextObjective removeFromExisting(ObjectiveContext context) {
+            treatments = listBuilder.build();
+            op = Operation.REMOVE_FROM_EXISTING;
+            this.context = context;
+            checkNotNull(appId, "Must supply an application id");
+            checkNotNull(id, "id cannot be null");
+            checkNotNull(type, "The type cannot be null");
+
+            return new DefaultNextObjective(this);
+        }
+
     }
+
 }

@@ -17,6 +17,7 @@ package org.onosproject.net.flowobjective;
 
 import com.google.common.annotations.Beta;
 import org.onosproject.core.ApplicationId;
+import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
 
 import java.util.Collection;
@@ -34,7 +35,7 @@ import java.util.Collection;
  * - Failover
  * - Simple
  *
- * These types will indicate to the driver what the intended behaviour is.
+ * These types will indicate to the driver what the intended behavior is.
  * For example, a broadcast next objective with a collection of output
  * treatments will indicate to a driver that all output actions are expected
  * to be executed simultaneously. The driver is then free to implement this
@@ -84,6 +85,16 @@ public interface NextObjective extends Objective {
     Type type();
 
     /**
+     * Auxiliary optional information provided to the device-driver.Typically
+     * conveys information about selectors (matches) that are intended to
+     * use this Next Objective.
+     *
+     * @return a selector intended to pass meta information to the device driver.
+     *         Value may be null if no meta information is provided.
+     */
+    TrafficSelector meta();
+
+    /**
      * A next step builder.
      */
     interface Builder extends Objective.Builder {
@@ -131,6 +142,14 @@ public interface NextObjective extends Objective {
         Builder withPriority(int priority);
 
         /**
+         * Set meta information related to this next objective.
+         *
+         * @param selector match conditions
+         * @return an objective builder
+         */
+        Builder setMeta(TrafficSelector selector);
+
+        /**
          * Builds the next objective that will be added.
          *
          * @return a next objective
@@ -162,6 +181,40 @@ public interface NextObjective extends Objective {
          */
         NextObjective remove(ObjectiveContext context);
 
+        /**
+         * Build the next objective that will be added, with {@link Operation}
+         * ADD_TO_EXISTING.
+         *
+         * @return a next objective
+         */
+        NextObjective addToExisting();
+
+        /**
+         * Build the next objective that will be removed, with {@link Operation}
+         * REMOVE_FROM_EXISTING.
+         *
+         * @return a next objective
+         */
+        NextObjective removeFromExisting();
+
+        /**
+         * Builds the next objective that will be added, with {@link Operation}
+         * ADD_TO_EXISTING. The context will be used to notify the calling application.
+         *
+         * @param context an objective context
+         * @return a next objective
+         */
+        NextObjective addToExisting(ObjectiveContext context);
+
+        /**
+         * Builds the next objective that will be removed, with {@link Operation}
+         * REMOVE_FROM_EXISTING. The context will be used to notify the calling application.
+         *
+         * @param context an objective context
+         * @return a next objective
+         */
+        NextObjective removeFromExisting(ObjectiveContext context);
+
     }
 
 }

@@ -21,7 +21,7 @@ import org.onosproject.core.ApplicationId;
 import java.util.Optional;
 
 /**
- * Base representation of an flow description.
+ * Base representation of a flow-objective description.
  */
 @Beta
 public interface Objective {
@@ -35,14 +35,30 @@ public interface Objective {
      */
     enum Operation {
         /**
-         * Adds the objective.
+         * Adds the objective. Can be used for any flow objective. For forwarding
+         * and filtering objectives, existing objectives with identical selector
+         * and priority fields (but different treatments or next) will be replaced.
+         * For next objectives, if modification is desired, ADD will not
+         * do anything - use ADD_TO_EXISTING.
          */
         ADD,
 
         /**
-         * Removes the objective.
+         * Removes the objective. Can be used for any flow objective.
          */
-        REMOVE
+        REMOVE,
+
+        /**
+         * Add to an existing Next Objective. Should not be used for any other
+         * objective.
+         */
+        ADD_TO_EXISTING,
+
+        /**
+         * Remove from an existing Next Objective. Should not be used for any
+         * other objective.
+         */
+        REMOVE_FROM_EXISTING
     }
 
     /**
@@ -129,6 +145,7 @@ public interface Objective {
          * @return an objective builder
          */
         Builder withPriority(int priority);
+
     }
 
 }

@@ -25,6 +25,7 @@ import java.util.Arrays;
 public class DefaultGroupKey implements GroupKey {
 
     private final byte[] key;
+    protected static final char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray();
 
     public DefaultGroupKey(byte[] key) {
         this.key = checkNotNull(key);
@@ -52,4 +53,20 @@ public class DefaultGroupKey implements GroupKey {
         return Arrays.hashCode(this.key);
     }
 
+    /**
+     * Returns a hex string representation of the byte array that is used
+     * as a group key. This solution was adapted from
+     * http://stackoverflow.com/questions/9655181/
+     */
+    @Override
+    public String toString() {
+        char[] hexChars = new char[key.length * 2];
+        for (int j = 0; j < key.length; j++) {
+            int v = key[j] & 0xFF;
+            hexChars[j * 2] = HEX_ARRAY[v >>> 4];
+            hexChars[j * 2 + 1] = HEX_ARRAY[v & 0x0F];
+        }
+        return "GroupKey:0x" + new String(hexChars);
+    }
+
 }
\ No newline at end of file

@@ -54,6 +54,7 @@ import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.Map;
+import java.util.Objects;
 import java.util.Set;
 import java.util.concurrent.ExecutorService;
 
@@ -226,10 +227,11 @@ public class FlowObjectiveManager implements FlowObjectiveService {
         if (fwd.nextId() != null &&
                 flowObjectiveStore.getNextGroup(fwd.nextId()) == null) {
             log.trace("Queuing forwarding objective for nextId {}", fwd.nextId());
-            if (pendingForwards.putIfAbsent(fwd.nextId(),
-                                            Sets.newHashSet(new PendingNext(deviceId, fwd))) != null) {
-                Set<PendingNext> pending = pendingForwards.get(fwd.nextId());
-                pending.add(new PendingNext(deviceId, fwd));
+            // TODO: change to computeIfAbsent
+            Set<PendingNext> pnext = pendingForwards.putIfAbsent(fwd.nextId(),
+                                         Sets.newHashSet(new PendingNext(deviceId, fwd)));
+            if (pnext != null) {
+                pnext.add(new PendingNext(deviceId, fwd));
             }
             return true;
         }
@@ -412,5 +414,26 @@ public class FlowObjectiveManager implements FlowObjectiveService {
         public ForwardingObjective forwardingObjective() {
             return fwd;
         }
+
+        @Override
+        public int hashCode() {
+            return Objects.hash(deviceId, fwd);
+        }
+
+        @Override
+        public boolean equals(final Object obj) {
+            if (this == obj) {
+                return true;
+            }
+            if (!(obj instanceof PendingNext)) {
+                return false;
+            }
+            final PendingNext other = (PendingNext) obj;
+            if (this.deviceId.equals(other.deviceId) &&
+                    this.fwd.equals(other.fwd)) {
+                return true;
+            }
+            return false;
+        }
     }
 }

@@ -348,9 +348,11 @@ public class DistributedGroupStore
     public void storeGroupDescription(GroupDescription groupDesc) {
         log.debug("In storeGroupDescription");
         // Check if a group is existing with the same key
-        if (getGroup(groupDesc.deviceId(), groupDesc.appCookie()) != null) {
-            log.warn("Group already exists with the same key {}",
-                     groupDesc.appCookie());
+        Group existingGroup = getGroup(groupDesc.deviceId(), groupDesc.appCookie());
+        if (existingGroup != null) {
+            log.warn("Group already exists with the same key {} in dev:{} with id:{}",
+                     groupDesc.appCookie(), groupDesc.deviceId(),
+                     Integer.toHexString(existingGroup.id().id()));
             return;
         }
 

@@ -368,6 +368,7 @@ public final class KryoNamespaces {
                     L2ModificationInstruction.ModVlanPcpInstruction.class,
                     L2ModificationInstruction.PopVlanInstruction.class,
                     L2ModificationInstruction.ModMplsLabelInstruction.class,
+                    L2ModificationInstruction.ModMplsBosInstruction.class,
                     L2ModificationInstruction.ModMplsTtlInstruction.class,
                     L2ModificationInstruction.ModTunnelIdInstruction.class,
                     L3ModificationInstruction.class,

@@ -18,15 +18,19 @@ package org.onosproject.driver.pipeline;
 import static org.slf4j.LoggerFactory.getLogger;
 
 import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
+import java.util.Deque;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 
+import org.onlab.packet.Ethernet;
 import org.onlab.packet.VlanId;
 import org.onosproject.core.ApplicationId;
 import org.onosproject.net.Port;
 import org.onosproject.net.PortNumber;
+import org.onosproject.net.behaviour.NextGroup;
 import org.onosproject.net.flow.DefaultFlowRule;
 import org.onosproject.net.flow.DefaultTrafficSelector;
 import org.onosproject.net.flow.DefaultTrafficTreatment;
@@ -35,8 +39,18 @@ import org.onosproject.net.flow.FlowRuleOperations;
 import org.onosproject.net.flow.FlowRuleOperationsContext;
 import org.onosproject.net.flow.TrafficSelector;
 import org.onosproject.net.flow.TrafficTreatment;
+import org.onosproject.net.flow.criteria.Criterion;
+import org.onosproject.net.flow.criteria.EthTypeCriterion;
+import org.onosproject.net.flow.criteria.IPCriterion;
+import org.onosproject.net.flow.criteria.MplsBosCriterion;
+import org.onosproject.net.flow.criteria.MplsCriterion;
 import org.onosproject.net.flow.criteria.PortCriterion;
 import org.onosproject.net.flow.criteria.VlanIdCriterion;
+import org.onosproject.net.flow.instructions.Instruction;
+import org.onosproject.net.flowobjective.ForwardingObjective;
+import org.onosproject.net.flowobjective.ObjectiveError;
+import org.onosproject.net.group.Group;
+import org.onosproject.net.group.GroupKey;
 import org.slf4j.Logger;
 
 
@@ -108,6 +122,81 @@ public class CpqdOFDPA2Pipeline extends OFDPA2Pipeline {
         return rules;
     }
 
+    @Override
+    protected Collection<FlowRule> processSpecific(ForwardingObjective fwd) {
+        TrafficSelector selector = fwd.selector();
+        EthTypeCriterion ethType =
+                (EthTypeCriterion) selector.getCriterion(Criterion.Type.ETH_TYPE);
+        if ((ethType == null) ||
+                (ethType.ethType().toShort() != Ethernet.TYPE_IPV4) &&
+                (ethType.ethType().toShort() != Ethernet.MPLS_UNICAST)) {
+            log.warn("processSpecific: Unsupported "
+                    + "forwarding objective criteraia");
+            fail(fwd, ObjectiveError.UNSUPPORTED);
+            return Collections.emptySet();
+        }
+
+        int forTableId = -1;
+        TrafficSelector.Builder filteredSelector = DefaultTrafficSelector.builder();
+        if (ethType.ethType().toShort() == Ethernet.TYPE_IPV4) {
+            filteredSelector.matchEthType(Ethernet.TYPE_IPV4)
+                .matchIPDst(((IPCriterion)
+                        selector.getCriterion(Criterion.Type.IPV4_DST)).ip());
+            forTableId = UNICAST_ROUTING_TABLE;
+            log.debug("processing IPv4 specific forwarding objective {} hash{} in dev:{}",
+                      fwd.id(), fwd.hashCode(), deviceId);
+        } else {
+            filteredSelector
+                .matchEthType(Ethernet.MPLS_UNICAST)
+                .matchMplsLabel(((MplsCriterion)
+                        selector.getCriterion(Criterion.Type.MPLS_LABEL)).label());
+            MplsBosCriterion bos = (MplsBosCriterion) selector
+                                        .getCriterion(Criterion.Type.MPLS_BOS);
+            if (bos != null) {
+                filteredSelector.matchMplsBos(bos.mplsBos());
+            }
+            forTableId = MPLS_TABLE_1;
+            log.debug("processing MPLS specific forwarding objective {} hash:{} in dev {}",
+                      fwd.id(), fwd.hashCode(), deviceId);
+        }
+
+        TrafficTreatment.Builder tb = DefaultTrafficTreatment.builder();
+        if (fwd.treatment() != null) {
+            for (Instruction i : fwd.treatment().allInstructions()) {
+                tb.add(i);
+            }
+        }
+
+        if (fwd.nextId() != null) {
+            NextGroup next = flowObjectiveStore.getNextGroup(fwd.nextId());
+            List<Deque<GroupKey>> gkeys = appKryo.deserialize(next.data());
+            // we only need the top level group's key to point the flow to it
+            Group group = groupService.getGroup(deviceId, gkeys.get(0).peekFirst());
+            if (group == null) {
+                log.warn("The group left!");
+                fail(fwd, ObjectiveError.GROUPMISSING);
+                return Collections.emptySet();
+            }
+            tb.deferred().group(group.id());
+        }
+        tb.transition(ACL_TABLE);
+        FlowRule.Builder ruleBuilder = DefaultFlowRule.builder()
+                .fromApp(fwd.appId())
+                .withPriority(fwd.priority())
+                .forDevice(deviceId)
+                .withSelector(filteredSelector.build())
+                .withTreatment(tb.build())
+                .forTable(forTableId);
+
+        if (fwd.permanent()) {
+            ruleBuilder.makePermanent();
+        } else {
+            ruleBuilder.makeTemporary(fwd.timeout());
+        }
+
+        return Collections.singletonList(ruleBuilder.build());
+    }
+
 
     @Override
     protected void initializePipeline() {